You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2015/04/24 02:25:55 UTC
[01/50] [abbrv] tez git commit: TEZ-2252. Tez UI: in graphical view
some of the sinks are hidden as they overlap (Sreenath Somarajapuram via
pramachandran)
Repository: tez
Updated Branches:
refs/heads/TEZ-2003 45ce66e0c -> 51b9db1a0 (forced update)
TEZ-2252. Tez UI: in graphical view some of the sinks are hidden as they overlap (Sreenath Somarajapuram via pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/cb6721b7
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/cb6721b7
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/cb6721b7
Branch: refs/heads/TEZ-2003
Commit: cb6721b7f515fdfe09163324d2e485e283be857a
Parents: f719c7b
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Mon Apr 13 16:59:55 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Mon Apr 13 16:59:55 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../components/dag-view/data-processor.js | 45 +++++++++++++++++---
.../scripts/components/dag-view/graph-view.js | 4 +-
3 files changed, 41 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/cb6721b7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 882f1f9..df8b06c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2252. Tez UI: in graphical view some of the sinks are hidden as they overlap
TEZ-2275. Tez UI: enable faster loading and caching of data in tables
TEZ-2234. Add API for statistics information - allow vertex managers to get
output size per source vertex
http://git-wip-us.apache.org/repos/asf/tez/blob/cb6721b7/tez-ui/src/main/webapp/app/scripts/components/dag-view/data-processor.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/dag-view/data-processor.js b/tez-ui/src/main/webapp/app/scripts/components/dag-view/data-processor.js
index 362a602..53bb7c9 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/dag-view/data-processor.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/dag-view/data-processor.js
@@ -167,6 +167,33 @@ App.DagViewComponent.dataProcessor = (function (){
};
/**
+ * Iterates the array in a symmetric order, from middle to outwards
+ * @param array {Array} Array to be iterated
+ * @param callback {Function} Function to be called for each item
+ * @return A new array created with value returned by callback
+ */
+ function centericMap(array, callback) {
+ var retArray = [],
+ length,
+ left, right;
+
+ if(array) {
+ length = array.length - 1,
+ left = length >> 1;
+
+ while(left >= 0) {
+ retArray[left] = callback(array[left]);
+ right = length - left;
+ if(right != left) {
+ retArray[right] = callback(array[right]);
+ }
+ left--;
+ }
+ }
+ return retArray;
+ }
+
+ /**
* Abstract class for all types of data nodes
*/
var DataNode = Em.Object.extend({
@@ -425,13 +452,12 @@ App.DagViewComponent.dataProcessor = (function (){
* @param vertex {VertexDataNode}
*/
function _treefyData(vertex, depth) {
- var children = [],
- parentChildren,
- inputDepth;
+ var children,
+ parentChildren;
depth++;
- vertex.ifForEach('inEdgeIds', function (edgeId) {
+ children = centericMap(vertex.get('inEdgeIds'), function (edgeId) {
var child = _data.vertices.get(_data.edges.get(edgeId).get('inputVertexName'));
if(!child.isSelfOrAncestor(vertex)) {
if(child.depth) {
@@ -446,10 +472,15 @@ App.DagViewComponent.dataProcessor = (function (){
}
}
child.setParent(vertex);
- children.push(_treefyData(child, depth));
+ return _treefyData(child, depth);
}
});
+ // Remove undefined entries
+ children = children.filter(function (child) {
+ return child;
+ });
+
vertex.setDepth(depth);
children.push.apply(children, vertex.get('inputs'));
@@ -481,7 +512,7 @@ App.DagViewComponent.dataProcessor = (function (){
// For a symmetric display of output nodes
if(childVertices && childVertices.length) {
- midIndex = Math.ceil(childVertices.length / 2);
+ midIndex = Math.floor(childVertices.length / 2);
if(childVertices.length % 2 == 0) {
midIndex--;
}
@@ -676,7 +707,7 @@ App.DagViewComponent.dataProcessor = (function (){
return "Sink vertex not found!";
}
- dummy._setChildren(_data.rootVertices.map(function (vertex) {
+ dummy._setChildren(centericMap(_data.rootVertices, function (vertex) {
return _treefyData(vertex, 2);
}));
http://git-wip-us.apache.org/repos/asf/tez/blob/cb6721b7/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js b/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
index c3c090b..6b54ce4 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
@@ -329,13 +329,13 @@ App.DagViewComponent.graphView = (function (){
if(rootChildCount % 2 == 0) {
dummyIndex = rootChildren.indexOf(_treeData.get('dummy'));
if(dummyIndex >= rootChildCount / 2) {
- for(var i = dummyIndex - 1; i >= 0; i--) {
+ for(var i = 0; i < dummyIndex; i++) {
rootChildren[i].x = rootChildren[i + 1].x,
rootChildren[i].y = rootChildren[i + 1].y;
}
}
else {
- for(var i = dummyIndex + 1; i < rootChildCount; i++) {
+ for(var i = rootChildCount - 1; i > dummyIndex; i--) {
rootChildren[i].x = rootChildren[i - 1].x,
rootChildren[i].y = rootChildren[i - 1].y;
}
[27/50] [abbrv] tez git commit: TEZ-2261. Should add diagnostics in
DAGAppMaster when recovery error happens (zjffdu)
Posted by ss...@apache.org.
TEZ-2261. Should add diagnostics in DAGAppMaster when recovery error happens (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/fe11c5e6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/fe11c5e6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/fe11c5e6
Branch: refs/heads/TEZ-2003
Commit: fe11c5e67334f1a08e2b586b68ee4840d8b62763
Parents: d91eb28
Author: Jeff Zhang <zj...@apache.org>
Authored: Thu Apr 23 09:46:54 2015 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Thu Apr 23 09:46:54 2015 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 35 +++++++++++++++-----
.../apache/tez/dag/app/MockDAGAppMaster.java | 20 ++++++++++-
.../tez/dag/app/TestMockDAGAppMaster.java | 28 ++++++++++++++++
4 files changed, 75 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/fe11c5e6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c32830e..af39092 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2261. Should add diagnostics in DAGAppMaster when recovery error happens
TEZ-2340. TestRecoveryParser fails
TEZ-2345. Tez UI: Enable cell level loading in all DAGs table
TEZ-2330. Create reconfigureVertex() API for input based initialization
http://git-wip-us.apache.org/repos/asf/tez/blob/fe11c5e6/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 3dd9e4c..8a914f6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -236,6 +236,7 @@ public class DAGAppMaster extends AbstractService {
private final Map<String, LocalResource> amResources = new HashMap<String, LocalResource>();
private final Map<String, LocalResource> cumulativeAdditionalResources = new HashMap<String, LocalResource>();
private final int maxAppAttempts;
+ private final List<String> diagnostics = new ArrayList<String>();
private boolean isLocal = false; //Local mode flag
@@ -383,6 +384,7 @@ public class DAGAppMaster extends AbstractService {
versionMismatchDiagnostics = "Incompatible versions found"
+ ", clientVersion=" + clientVersion
+ ", AMVersion=" + dagVersionInfo.getVersion();
+ addDiagnostic(versionMismatchDiagnostics);
if (disableVersionCheck) {
LOG.warn("Ignoring client-AM version mismatch as check disabled. "
+ versionMismatchDiagnostics);
@@ -491,7 +493,7 @@ public class DAGAppMaster extends AbstractService {
addIfService(containerLauncher, true);
dispatcher.register(NMCommunicatorEventType.class, containerLauncher);
- historyEventHandler = new HistoryEventHandler(context);
+ historyEventHandler = createHistoryEventHandler(context);
addIfService(historyEventHandler, true);
this.sessionTimeoutInterval = 1000 * amConf.getInt(
@@ -563,6 +565,11 @@ public class DAGAppMaster extends AbstractService {
return new AsyncDispatcher("Central");
}
+ @VisibleForTesting
+ protected HistoryEventHandler createHistoryEventHandler(AppContext appContext) {
+ return new HistoryEventHandler(appContext);
+ }
+
/**
* Exit call. Just in a function call to enable testing.
*/
@@ -624,8 +631,10 @@ public class DAGAppMaster extends AbstractService {
lastDAGCompletionTime = clock.getTime();
_updateLoggers(currentDAG, "_post");
if (this.historyEventHandler.hasRecoveryFailed()) {
- LOG.warn("Recovery had a fatal error, shutting down session after" +
- " DAG completion");
+ String recoveryErrorMsg = "Recovery had a fatal error, shutting down session after" +
+ " DAG completion";
+ LOG.warn(recoveryErrorMsg);
+ addDiagnostic(recoveryErrorMsg);
sessionStopped.set(true);
}
switch(finishEvt.getDAGState()) {
@@ -1071,22 +1080,32 @@ public class DAGAppMaster extends AbstractService {
return state;
}
+ private void addDiagnostic(String diag) {
+ synchronized (diagnostics) {
+ diagnostics.add(diag);
+ }
+ }
+
public List<String> getDiagnostics() {
- if (versionMismatch) {
- return Collections.singletonList(versionMismatchDiagnostics);
+ // always create new diagnostics to return
+ // This is to avoid the case that this method is called multiple times and diagnostics is accumulated.
+ List<String> diagResult = new ArrayList<String>();
+ synchronized (diagnostics) {
+ diagResult.addAll(this.diagnostics);
}
+
if (!isSession) {
if(currentDAG != null) {
- return currentDAG.getDiagnostics();
+ diagResult.addAll(currentDAG.getDiagnostics());
}
} else {
- return Collections.singletonList("Session stats:"
+ diagResult.add("Session stats:"
+ "submittedDAGs=" + submittedDAGs.get()
+ ", successfulDAGs=" + successfulDAGs.get()
+ ", failedDAGs=" + failedDAGs.get()
+ ", killedDAGs=" + killedDAGs.get());
}
- return null;
+ return diagResult;
}
public float getProgress() {
http://git-wip-us.apache.org/repos/asf/tez/blob/fe11c5e6/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index 92dfdb5..18286b5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -59,6 +59,7 @@ import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
import org.apache.tez.dag.app.rm.container.AMContainerEvent;
import org.apache.tez.dag.app.rm.container.AMContainerEventLaunched;
import org.apache.tez.dag.app.rm.container.AMContainerEventType;
+import org.apache.tez.dag.history.HistoryEventHandler;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.runtime.api.events.CompositeDataMovementEvent;
@@ -92,6 +93,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
boolean initFailFlag;
boolean startFailFlag;
boolean sendDMEvents;
+ boolean recoveryFatalError = false;
CountersDelegate countersDelegate;
StatisticsDelegate statsDelegate;
long launcherSleepTime = 1;
@@ -453,6 +455,17 @@ public class MockDAGAppMaster extends DAGAppMaster {
}
}
+ public class MockHistoryEventHandler extends HistoryEventHandler {
+
+ public MockHistoryEventHandler(AppContext context) {
+ super(context);
+ }
+
+ @Override
+ public boolean hasRecoveryFailed() {
+ return recoveryFatalError;
+ }
+ }
public class MockDAGAppMasterShutdownHandler extends DAGAppMasterShutdownHandler {
public AtomicInteger shutdownInvoked = new AtomicInteger(0);
@@ -498,7 +511,12 @@ public class MockDAGAppMaster extends DAGAppMaster {
throws UnknownHostException {
return containerLauncher;
}
-
+
+ @Override
+ protected HistoryEventHandler createHistoryEventHandler(AppContext appContext) {
+ return new MockHistoryEventHandler(appContext);
+ }
+
public MockContainerLauncher getContainerLauncher() {
return containerLauncher;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/fe11c5e6/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
index db1d632..33dd18d 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
@@ -887,4 +887,32 @@ public class TestMockDAGAppMaster {
}
}
}
+
+ @Test(timeout = 5000)
+ public void testDAGFinishedRecoveryError() throws Exception {
+ TezConfiguration tezconf = new TezConfiguration(defaultConf);
+
+ MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
+ tezClient.start();
+
+ MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
+ mockApp.recoveryFatalError = true;
+ MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
+ mockLauncher.startScheduling(true);
+
+ DAG dag = DAG.create("test");
+ Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
+ dag.addVertex(vA);
+
+ DAGClient dagClient = tezClient.submitDAG(dag);
+ dagClient.waitForCompletion();
+ while(!mockApp.getShutdownHandler().wasShutdownInvoked()) {
+ Thread.sleep(100);
+ }
+ Assert.assertEquals(DAGState.SUCCEEDED, mockApp.getContext().getCurrentDAG().getState());
+ Assert.assertEquals(DAGAppMasterState.FAILED, mockApp.getState());
+ Assert.assertTrue(StringUtils.join(mockApp.getDiagnostics(),",")
+ .contains("Recovery had a fatal error, shutting down session after" +
+ " DAG completion"));
+ }
}
[45/50] [abbrv] tez git commit: TEZ-2283. Fixes after rebase 04/07.
(sseth)
Posted by ss...@apache.org.
TEZ-2283. Fixes after rebase 04/07. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/623e8ac5
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/623e8ac5
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/623e8ac5
Branch: refs/heads/TEZ-2003
Commit: 623e8ac5a22afb3625ed5a3d372372cfbf371f67
Parents: b828d15
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Apr 7 13:13:30 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:47:33 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../java/org/apache/tez/dag/app/rm/TestContainerReuse.java | 8 ++++----
.../org/apache/tez/service/impl/ContainerRunnerImpl.java | 4 +---
3 files changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/623e8ac5/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index d51686d..6a4399c 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -12,5 +12,6 @@ ALL CHANGES:
TEZ-2175. Task priority should be available to the TaskCommunicator plugin.
TEZ-2187. Allow TaskCommunicators to report failed / killed attempts.
TEZ-2241. Miscellaneous fixes after last reabse.
+ TEZ-2283. Fixes after rebase 04/07.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/623e8ac5/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
index d45346a..8b4ff07 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
@@ -1242,9 +1242,9 @@ public class TestContainerReuse {
assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
// Task assigned to container completed successfully. Container should be re-used.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta111), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta111), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta112), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
@@ -1254,9 +1254,9 @@ public class TestContainerReuse {
// Task assigned to container completed successfully.
// Verify reuse across hosts.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta112), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta112), eq(true), eq((TaskAttemptEndReason)null));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
eventHandler.reset();
http://git-wip-us.apache.org/repos/asf/tez/blob/623e8ac5/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
index e7c02c8..f3fc442 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
@@ -69,10 +69,8 @@ import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl;
import org.apache.tez.runtime.task.TezChild;
import org.apache.tez.runtime.task.TezChild.ContainerExecutionResult;
import org.apache.tez.shufflehandler.ShuffleHandler;
-import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
-import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.TaskSpecProto;
import org.apache.tez.util.ProtoConverters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -446,7 +444,7 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
request.getContainerIdString());
taskRunner = new TezTaskRunner(conf, taskUgi, localDirs,
- ProtoConverters.getTaskSpecfromProto(request.getTaskSpec()), umbilical,
+ ProtoConverters.getTaskSpecfromProto(request.getTaskSpec()),
request.getAppAttemptNumber(),
serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor, objectRegistry,
pid,
[20/50] [abbrv] tez git commit: TEZ-2313. Regression in handling
obsolete events in ShuffleScheduler (rbalamohan)
Posted by ss...@apache.org.
TEZ-2313. Regression in handling obsolete events in ShuffleScheduler (rbalamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/decb4191
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/decb4191
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/decb4191
Branch: refs/heads/TEZ-2003
Commit: decb4191bbada28749483b1b5af837fc87aff8bc
Parents: 44046f8
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Tue Apr 21 11:10:33 2015 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Tue Apr 21 11:10:33 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../common/shuffle/orderedgrouped/ShuffleScheduler.java | 8 +++++---
.../TestShuffleInputEventHandlerOrderedGrouped.java | 4 ++++
3 files changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/decb4191/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 64b3561..bd9ff6d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2313. Regression in handling obsolete events in ShuffleScheduler.
TEZ-2212. Notify components on DAG completion.
TEZ-2328. Add tez.runtime.sorter.class & rename tez.runtime.sort.threads
to tez.runtime.pipelined.sorter.sort.threads.
http://git-wip-us.apache.org/repos/asf/tez/blob/decb4191/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java
index d0b6346..a3d79ae 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java
@@ -349,6 +349,10 @@ class ShuffleScheduler {
+ ", newAttemptNum=" + input.getAttemptNumber()));
return false;
}
+
+ if (eventInfo == null) {
+ shuffleInfoEventsMap.put(input.getInputIdentifier(), new ShuffleEventInfo(input));
+ }
}
return true;
}
@@ -523,9 +527,6 @@ class ShuffleScheduler {
if (!validateInputAttemptForPipelinedShuffle(srcAttempt)) {
return;
}
- if (shuffleInfoEventsMap.get(srcAttempt.getInputIdentifier()) == null) {
- shuffleInfoEventsMap.put(srcAttempt.getInputIdentifier(), new ShuffleEventInfo(srcAttempt));
- }
host.addKnownMap(srcAttempt);
pathToIdentifierMap.put(
@@ -542,6 +543,7 @@ class ShuffleScheduler {
// The incoming srcAttempt does not contain a path component.
LOG.info("Adding obsolete input: " + srcAttempt);
if (shuffleInfoEventsMap.containsKey(srcAttempt.getInputIdentifier())) {
+ //Pipelined shuffle case (where shuffleInfoEventsMap gets populated).
//Fail fast here.
shuffle.reportException(new IOException(srcAttempt + " is marked as obsoleteInput, but it "
+ "exists in shuffleInfoEventMap. Some data could have been already merged "
http://git-wip-us.apache.org/repos/asf/tez/blob/decb4191/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java
index 460db01..eed9fd8 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java
@@ -236,6 +236,8 @@ public class TestShuffleInputEventHandlerOrderedGrouped {
PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0);
verify(scheduler, times(1)).addKnownMapOutput(eq(HOST), eq(PORT), eq(1), eq(baseUri), eq(id1));
+ assertTrue("Shuffle info events should not be empty for pipelined shuffle",
+ !scheduler.shuffleInfoEventsMap.isEmpty());
//Attempt #0 comes up. When processing this, it should report exception
attemptNum = 0;
@@ -263,6 +265,8 @@ public class TestShuffleInputEventHandlerOrderedGrouped {
int partitionId = srcIdx;
verify(scheduler).addKnownMapOutput(eq(HOST), eq(PORT), eq(partitionId),
eq(baseUri), eq(expectedIdentifier));
+ assertTrue("Shuffle info events should be empty for regular shuffle codepath",
+ scheduler.shuffleInfoEventsMap.isEmpty());
}
@Test(timeout = 5000)
[21/50] [abbrv] tez git commit: Tez UI: Equip basic-ember-table's
cell level loading for all use cases in all DAGs table (Sreenath
Somarajapuram via pramachandran)
Posted by ss...@apache.org.
Tez UI: Equip basic-ember-table's cell level loading for all use cases in all DAGs table (Sreenath Somarajapuram via pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/87aac127
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/87aac127
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/87aac127
Branch: refs/heads/TEZ-2003
Commit: 87aac1270eb12b801ba326d5f53ba054c62ed58c
Parents: decb419
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Tue Apr 21 16:38:13 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Tue Apr 21 16:39:01 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
tez-ui/src/main/webapp/app/scripts/app.js | 1 +
.../scripts/components/basic-table/cell-view.js | 20 +++++++-------
.../components/basic-table/basic-cell.hbs | 6 ++++-
.../basic-table/bounded-basic-cell.hbs | 27 +++++++++++++++++++
.../components/basic-table/linked-cell.hbs | 16 +++++++----
.../components/basic-table/logs-cell.hbs | 28 +++++++++-----------
.../components/basic-table/pagination-view.hbs | 10 ++++---
.../components/basic-table/status-cell.hbs | 20 +++++++-------
.../basic-table/task-actions-cell.hbs | 12 ++++-----
.../basic-table/vertex-configurations-cell.hbs | 14 +++++-----
11 files changed, 96 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bd9ff6d..4ba2f2f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2344. Tez UI: Equip basic-ember-table's cell level loading for all use cases in all DAGs table
TEZ-2313. Regression in handling obsolete events in ShuffleScheduler.
TEZ-2212. Notify components on DAG completion.
TEZ-2328. Add tez.runtime.sorter.class & rename tez.runtime.sort.threads
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/tez-ui/src/main/webapp/app/scripts/app.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/app.js b/tez-ui/src/main/webapp/app/scripts/app.js
index 1d8036d..37aa563 100644
--- a/tez-ui/src/main/webapp/app/scripts/app.js
+++ b/tez-ui/src/main/webapp/app/scripts/app.js
@@ -80,6 +80,7 @@ App.ready = function () {
},
});
+ App.DagVertexAdapter =
App.VertexAdapter = App.ApplicationAdapter.extend({
_setInputs: function (store, data) {
var dagId = Ember.get(data, 'primaryfilters.TEZ_DAG_ID.0'),
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/tez-ui/src/main/webapp/app/scripts/components/basic-table/cell-view.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/basic-table/cell-view.js b/tez-ui/src/main/webapp/app/scripts/components/basic-table/cell-view.js
index 4d50cf1..1506577 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/basic-table/cell-view.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/basic-table/cell-view.js
@@ -16,28 +16,28 @@
* limitations under the License.
*/
-var ObjectPromiseController = Ember.ObjectController.extend(Ember.PromiseProxyMixin),
- naCellContent = {
- toString: function (){
- return 'Not available!';
- },
- _notAvailable: true
- };
+var ObjectPromiseController = Ember.ObjectController.extend(Ember.PromiseProxyMixin);
App.BasicTableComponent.CellView = Ember.View.extend({
templateName: 'components/basic-table/basic-cell',
classNames: ['cell-content'],
+ _normalizeContent: function (content) {
+ return content && typeof content == 'object' ? content : {
+ displayText: content
+ };
+ },
+
cellContent: function () {
var cellContent = this.get('column').getCellContent(this.get('row'));
if(cellContent && $.isFunction(cellContent.then)) {
- cellContent = ObjectPromiseController.create({
- promise: cellContent
+ return ObjectPromiseController.create({
+ promise: cellContent.then(this._normalizeContent)
});
}
- return cellContent || naCellContent;
+ return this._normalizeContent(cellContent);
}.property('row', 'column')
});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/tez-ui/src/main/webapp/app/templates/components/basic-table/basic-cell.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/basic-table/basic-cell.hbs b/tez-ui/src/main/webapp/app/templates/components/basic-table/basic-cell.hbs
index 370637b..e7ea6b7 100644
--- a/tez-ui/src/main/webapp/app/templates/components/basic-table/basic-cell.hbs
+++ b/tez-ui/src/main/webapp/app/templates/components/basic-table/basic-cell.hbs
@@ -16,4 +16,8 @@
* limitations under the License.
}}
-{{unbound view.cellContent}}
\ No newline at end of file
+{{#unbound unless view.cellContent.displayText}}
+ <span class="message">Not Available!</span>
+{{else}}
+ {{unbound view.cellContent.displayText}}
+{{/unbound}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/tez-ui/src/main/webapp/app/templates/components/basic-table/bounded-basic-cell.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/basic-table/bounded-basic-cell.hbs b/tez-ui/src/main/webapp/app/templates/components/basic-table/bounded-basic-cell.hbs
new file mode 100644
index 0000000..6fcbe60
--- /dev/null
+++ b/tez-ui/src/main/webapp/app/templates/components/basic-table/bounded-basic-cell.hbs
@@ -0,0 +1,27 @@
+{{!
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+}}
+
+{{#if view.cellContent.isPending}}
+ <i class="waiting"></i>
+{{else}}
+ {{#if view.cellContent.displayText}}
+ {{view.cellContent.displayText}}
+ {{else}}
+ <span class="message">Not Available!</span>
+ {{/if}}
+{{/if}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/tez-ui/src/main/webapp/app/templates/components/basic-table/linked-cell.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/basic-table/linked-cell.hbs b/tez-ui/src/main/webapp/app/templates/components/basic-table/linked-cell.hbs
index 3297ad7..5800b6e 100644
--- a/tez-ui/src/main/webapp/app/templates/components/basic-table/linked-cell.hbs
+++ b/tez-ui/src/main/webapp/app/templates/components/basic-table/linked-cell.hbs
@@ -16,10 +16,16 @@
* limitations under the License.
}}
-{{#if view.cellContent._notAvailable}}
- {{view.cellContent}}
+{{#if view.cellContent.isPending}}
+ <i class="waiting"></i>
{{else}}
- {{#link-to view.cellContent.linkTo view.cellContent.entityId}}
+ {{#if view.cellContent.linkTo}}
+ {{#link-to view.cellContent.linkTo view.cellContent.entityId}}
+ {{view.cellContent.displayText}}
+ {{/link-to}}
+ {{else}}{{#if view.cellContent.displayText}}
{{view.cellContent.displayText}}
- {{/link-to}}
-{{/if}}
\ No newline at end of file
+ {{else}}
+ <span class="message">Not Available!</span>
+ {{/if}}{{/if}}
+{{/if}}
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/tez-ui/src/main/webapp/app/templates/components/basic-table/logs-cell.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/basic-table/logs-cell.hbs b/tez-ui/src/main/webapp/app/templates/components/basic-table/logs-cell.hbs
index 092b6ee..a4ab4a1 100644
--- a/tez-ui/src/main/webapp/app/templates/components/basic-table/logs-cell.hbs
+++ b/tez-ui/src/main/webapp/app/templates/components/basic-table/logs-cell.hbs
@@ -16,20 +16,16 @@
* limitations under the License.
}}
-<span class="ember-table-content">
- {{#if view.cellContent.isPending}}
- <i class="waiting absolute"></i>
- {{else}}
- {{#if view.cellContent._notAvailable}}
- {{view.cellContent}}
- {{else}}
- {{#if view.cellContent.viewUrl}}
- <a target="_blank" href="//{{unbound view.cellContent.viewUrl}}">View</a>
-
- {{/if}}
- {{#if view.cellContent.downloadUrl}}
- <a target="_blank" href="{{unbound view.cellContent.downloadUrl}}?start=0" download type="application/octet-stream">Download</a>
- {{/if}}
- {{/if}}
+{{#if view.cellContent.isPending}}
+ <i class="waiting"></i>
+{{else}}
+ {{#if view.cellContent.viewUrl}}
+ <a target="_blank" href="//{{unbound view.cellContent.viewUrl}}">View</a>
+
{{/if}}
-</span>
+ {{#if view.cellContent.downloadUrl}}
+ <a target="_blank" href="{{unbound view.cellContent.downloadUrl}}?start=0" download type="application/octet-stream">Download</a>
+ {{else}}{{#unless view.cellContent.viewUrl}}
+ <span class="message">Not Available!</span>
+ {{/unless}}{{/if}}
+{{/if}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/tez-ui/src/main/webapp/app/templates/components/basic-table/pagination-view.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/basic-table/pagination-view.hbs b/tez-ui/src/main/webapp/app/templates/components/basic-table/pagination-view.hbs
index 808fe9d..22ee22b 100644
--- a/tez-ui/src/main/webapp/app/templates/components/basic-table/pagination-view.hbs
+++ b/tez-ui/src/main/webapp/app/templates/components/basic-table/pagination-view.hbs
@@ -18,16 +18,18 @@
<ul class="page-list">
<li {{bind-attr class="view.atFirst::clickable"}} {{action 'changePage' 1}}>
- First page
+ First
</li>
{{#each page in view._possiblePages}}
<li {{bind-attr class="page.isCurrent:is-current:clickable"}} {{action 'changePage' page.pageNum}}>
{{page.pageNum}}
</li>
{{/each}}
- <li {{bind-attr class="view.atLast::clickable"}} {{action 'changePage' view.totalPages}}>
- Last page - {{view.totalPages}}
- </li>
+ {{#unless view.hideLast}}
+ <li {{bind-attr class="view.atLast::clickable"}} {{action 'changePage' view.totalPages}}>
+ Last - {{view.totalPages}}
+ </li>
+ {{/unless}}
</ul>
<div class='row-select'>
<div>Rows</div>
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/tez-ui/src/main/webapp/app/templates/components/basic-table/status-cell.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/basic-table/status-cell.hbs b/tez-ui/src/main/webapp/app/templates/components/basic-table/status-cell.hbs
index a7caa35..7ff73ee 100644
--- a/tez-ui/src/main/webapp/app/templates/components/basic-table/status-cell.hbs
+++ b/tez-ui/src/main/webapp/app/templates/components/basic-table/status-cell.hbs
@@ -16,14 +16,16 @@
* limitations under the License.
}}
-{{#if view.cellContent._notAvailable}}
- {{view.cellContent}}
+{{#if view.cellContent.isPending}}
+ <i class="waiting"></i>
{{else}}
- <span class="ember-table-content">
+ {{#if view.cellContent.status}}
<i {{bind-attr class=":task-status view.cellContent.statusIcon"}}></i>
- {{view.cellContent.status}}
- {{#if view.cellContent.progress}}
- {{bs-badge content=view.cellContent.progress}}
- {{/if}}
- </span>
-{{/if}}
\ No newline at end of file
+ <span class="status-msg">{{view.cellContent.status}}</span>
+ {{/if}}
+ {{#if view.cellContent.progress}}
+ {{bs-badge content=view.cellContent.progress}}
+ {{else}}{{#unless view.cellContent.status}}
+ <span class="message">Not Available!</span>
+ {{/unless}}{{/if}}
+{{/if}}
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/tez-ui/src/main/webapp/app/templates/components/basic-table/task-actions-cell.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/basic-table/task-actions-cell.hbs b/tez-ui/src/main/webapp/app/templates/components/basic-table/task-actions-cell.hbs
index b914b82..e671cf3 100644
--- a/tez-ui/src/main/webapp/app/templates/components/basic-table/task-actions-cell.hbs
+++ b/tez-ui/src/main/webapp/app/templates/components/basic-table/task-actions-cell.hbs
@@ -16,11 +16,9 @@
* limitations under the License.
}}
-{{#if view.cellContent._notAvailable}}
- {{view.cellContent}}
+{{#if view.cellContent.displayText}}
+ {{#link-to "task.counters" view.cellContent.displayText}}counters{{/link-to}}
+ {{#link-to "task.attempts" view.cellContent.displayText}}attempts{{/link-to}}
{{else}}
- <span class="ember-table-content">
- {{#link-to "task.counters" view.cellContent}}counters{{/link-to}}
- {{#link-to "task.attempts" view.cellContent}}attempts{{/link-to}}
- </span>
-{{/if}}
\ No newline at end of file
+ <span class="message">Not Available!</span>
+{{/if}}
http://git-wip-us.apache.org/repos/asf/tez/blob/87aac127/tez-ui/src/main/webapp/app/templates/components/basic-table/vertex-configurations-cell.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/basic-table/vertex-configurations-cell.hbs b/tez-ui/src/main/webapp/app/templates/components/basic-table/vertex-configurations-cell.hbs
index 17fb7e3..85f9eb1 100644
--- a/tez-ui/src/main/webapp/app/templates/components/basic-table/vertex-configurations-cell.hbs
+++ b/tez-ui/src/main/webapp/app/templates/components/basic-table/vertex-configurations-cell.hbs
@@ -16,16 +16,16 @@
* limitations under the License.
}}
-{{#if view.cellContent._notAvailable}}
- {{view.cellContent}}
+{{#if view.cellContent.isPending}}
+ <i class="waiting"></i>
{{else}}
{{#if view.cellContent.linkToAdditionals}}
- {{#link-to 'vertex.additionals' view.cellContent.vertexId class='ember-table-content'}}View sources & sinks{{/link-to}}
+ {{#link-to 'vertex.additionals' view.cellContent.vertexId}}View sources & sinks{{/link-to}}
{{else}}{{#if view.cellContent.inputId}}
- {{#link-to 'input.configs' view.cellContent.vertexId view.cellContent.inputId class='ember-table-content'}}View source configs{{/link-to}}
+ {{#link-to 'input.configs' view.cellContent.vertexId view.cellContent.inputId}}View source configs{{/link-to}}
{{else}}{{#if view.cellContent.outputId}}
- {{#link-to 'output.configs' view.cellContent.vertexId view.cellContent.outputId class='ember-table-content'}}View sink configs{{/link-to}}
+ {{#link-to 'output.configs' view.cellContent.vertexId view.cellContent.outputId}}View sink configs{{/link-to}}
{{else}}
- <span class='ember-table-content'>No source or sink</span>
+ <span class="message">Not Available!</span>
{{/if}}{{/if}}{{/if}}
-{{/if}}
\ No newline at end of file
+{{/if}}
[04/50] [abbrv] tez git commit: TEZ-2033. Update TestOrderedWordCount
to add processor configs as history text and use MR configs correctly.
(hitesh)
Posted by ss...@apache.org.
TEZ-2033. Update TestOrderedWordCount to add processor configs as history text and use MR configs correctly. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/beb77856
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/beb77856
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/beb77856
Branch: refs/heads/TEZ-2003
Commit: beb77856eb6eadf11001df28975f30ed8752c7f0
Parents: fa72bdc
Author: Hitesh Shah <hi...@apache.org>
Authored: Tue Apr 14 11:31:17 2015 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Tue Apr 14 11:31:17 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../examples/TestOrderedWordCount.java | 35 ++++++++++----------
2 files changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/beb77856/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b47cf7d..cdc48e1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,8 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2033. Update TestOrderedWordCount to add processor configs as history text
+ and use MR configs correctly
TEZ-2318. Tez UI: source and sink page is broken as they are not populated.
TEZ-2016. Tez UI: Dag View Fit and Finish
TEZ-2252. Tez UI: in graphical view some of the sinks are hidden as they overlap
http://git-wip-us.apache.org/repos/asf/tez/blob/beb77856/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
index 92e3a53..4a05dab 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
@@ -162,8 +162,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
Configuration mapStageConf = new JobConf(conf);
mapStageConf.set(MRJobConfig.MAP_CLASS_ATTR,
TokenizerMapper.class.getName());
-
- MRHelpers.translateMRConfToTez(mapStageConf);
+ MRHelpers.translateMRConfToTez(mapStageConf, !useMRSettings);
Configuration iReduceStageConf = new JobConf(conf);
// TODO replace with auto-reduce parallelism
@@ -174,7 +173,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
iReduceStageConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS,
IntWritable.class.getName());
iReduceStageConf.setBoolean("mapred.mapper.new-api", true);
- MRHelpers.translateMRConfToTez(iReduceStageConf);
+ MRHelpers.translateMRConfToTez(iReduceStageConf, !useMRSettings);
Configuration finalReduceConf = new JobConf(conf);
finalReduceConf.setInt(MRJobConfig.NUM_REDUCES, 1);
@@ -182,7 +181,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
MyOrderByNoOpReducer.class.getName());
finalReduceConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, IntWritable.class.getName());
finalReduceConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, Text.class.getName());
- MRHelpers.translateMRConfToTez(finalReduceConf);
+ MRHelpers.translateMRConfToTez(finalReduceConf, !useMRSettings);
MRHelpers.configureMRApiUsage(mapStageConf);
MRHelpers.configureMRApiUsage(iReduceStageConf);
@@ -190,19 +189,22 @@ public class TestOrderedWordCount extends Configured implements Tool {
List<Vertex> vertices = new ArrayList<Vertex>();
- ByteArrayOutputStream outputStream = new ByteArrayOutputStream(4096);
- mapStageConf.writeXml(outputStream);
- String mapStageHistoryText = new String(outputStream.toByteArray(), "UTF-8");
+ String mapStageHistoryText = TezUtils.convertToHistoryText("Initial Tokenizer Vertex",
+ mapStageConf);
DataSourceDescriptor dsd;
if (generateSplitsInClient) {
mapStageConf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR,
TextInputFormat.class.getName());
mapStageConf.set(FileInputFormat.INPUT_DIR, inputPath);
mapStageConf.setBoolean("mapred.mapper.new-api", true);
- dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(mapStageConf, stagingDir, true);
+ dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(mapStageConf, stagingDir,
+ true);
} else {
- dsd = MRInputLegacy.createConfigBuilder(mapStageConf, TextInputFormat.class, inputPath).build();
+ dsd = MRInputLegacy.createConfigBuilder(mapStageConf, TextInputFormat.class,
+ inputPath).build();
}
+ dsd.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(
+ "HDFS Input " + inputPath, mapStageConf));
Map<String, String> mapEnv = Maps.newHashMap();
MRHelpers.updateEnvBasedOnMRTaskEnv(mapStageConf, mapEnv, true);
@@ -227,10 +229,8 @@ public class TestOrderedWordCount extends Configured implements Tool {
.addDataSource("MRInput", dsd);
vertices.add(mapVertex);
- ByteArrayOutputStream iROutputStream = new ByteArrayOutputStream(4096);
- iReduceStageConf.writeXml(iROutputStream);
- String iReduceStageHistoryText = new String(iROutputStream.toByteArray(), "UTF-8");
-
+ String iReduceStageHistoryText = TezUtils.convertToHistoryText("Intermediate Summation Vertex",
+ iReduceStageConf);
ProcessorDescriptor iReduceProcessorDescriptor = ProcessorDescriptor.create(
ReduceProcessor.class.getName())
.setUserPayload(TezUtils.createUserPayloadFromConf(iReduceStageConf))
@@ -249,9 +249,8 @@ public class TestOrderedWordCount extends Configured implements Tool {
intermediateVertex.addTaskLocalFiles(commonLocalResources);
vertices.add(intermediateVertex);
- ByteArrayOutputStream finalReduceOutputStream = new ByteArrayOutputStream(4096);
- finalReduceConf.writeXml(finalReduceOutputStream);
- String finalReduceStageHistoryText = new String(finalReduceOutputStream.toByteArray(), "UTF-8");
+ String finalReduceStageHistoryText = TezUtils.convertToHistoryText("Final Sorter Vertex",
+ finalReduceConf);
UserPayload finalReducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
Vertex finalReduceVertex;
@@ -272,6 +271,8 @@ public class TestOrderedWordCount extends Configured implements Tool {
finalReduceVertex.addDataSink("MROutput",
MROutputLegacy.createConfigBuilder(finalReduceConf, TextOutputFormat.class, outputPath)
.build());
+ finalReduceVertex.getDataSinks().get(0).getOutputDescriptor().setHistoryText(
+ TezUtils.convertToHistoryText("HDFS Output " + outputPath, finalReduceConf));
vertices.add(finalReduceVertex);
DAG dag = DAG.create("OrderedWordCount" + dagIndex);
@@ -546,7 +547,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
}
public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new TestOrderedWordCount(), args);
+ int res = ToolRunner.run(new TezConfiguration(), new TestOrderedWordCount(), args);
System.exit(res);
}
}
[18/50] [abbrv] tez git commit: TEZ-2328. Add
tez.runtime.sorter.class & rename tez.runtime.sort.threads to
tez.runtime.pipelined.sorter.sort.threads. (hitesh)
Posted by ss...@apache.org.
TEZ-2328. Add tez.runtime.sorter.class & rename tez.runtime.sort.threads to tez.runtime.pipelined.sorter.sort.threads. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/db0a50c5
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/db0a50c5
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/db0a50c5
Branch: refs/heads/TEZ-2003
Commit: db0a50c5eaed0ab19a762658bc661d792d8b31a7
Parents: 57c62f1
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon Apr 20 11:28:57 2015 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon Apr 20 11:28:57 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../library/api/TezRuntimeConfiguration.java | 29 +++++++--
.../common/sort/impl/PipelinedSorter.java | 4 +-
.../conf/OrderedPartitionedKVOutputConfig.java | 42 ++++++++++++-
.../output/OrderedPartitionedKVOutput.java | 34 +++++++---
.../api/TestTezRuntimeConfiguration.java | 2 +-
.../common/shuffle/TestShuffleUtils.java | 1 -
.../common/sort/impl/TestPipelinedSorter.java | 3 +-
.../sort/impl/dflt/TestDefaultSorter.java | 3 +-
.../TestOrderedPartitionedKVOutputConfig.java | 1 +
.../library/output/TestOnFileSortedOutput.java | 65 ++++++++++++++------
11 files changed, 143 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9c86434..c44824c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,8 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2328. Add tez.runtime.sorter.class & rename tez.runtime.sort.threads
+ to tez.runtime.pipelined.sorter.sort.threads.
TEZ-2333. Enable local fetch optimization by default.
TEZ-2310. Deadlock caused by StateChangeNotifier sending notifications on
thread holding locks
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
index b1341f6..a818de8 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
@@ -17,8 +17,6 @@
*/
package org.apache.tez.runtime.library.api;
-
-
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -32,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl;
/**
* Meant for user configurable job properties.
@@ -62,6 +61,11 @@ public class TezRuntimeConfiguration {
private static List<String> unmodifiableAllowedPrefixes;
+ static {
+ Configuration.addDeprecation("tez.runtime.sort.threads",
+ TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS);
+ }
+
/**
* Configuration key to enable/disable IFile readahead.
*/
@@ -113,9 +117,21 @@ public class TezRuntimeConfiguration {
public static final int TEZ_RUNTIME_COMBINE_MIN_SPILLS_DEFAULT = 3;
- public static final String TEZ_RUNTIME_SORT_THREADS = TEZ_RUNTIME_PREFIX +
- "sort.threads";
- public static final int TEZ_RUNTIME_SORT_THREADS_DEFAULT = 2;
+ /**
+ * String value.
+ * Which sorter implementation to use.
+ * Valid values:
+ * - LEGACY
+ * - PIPELINED ( default )
+ * {@link org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl}
+ */
+ public static final String TEZ_RUNTIME_SORTER_CLASS = TEZ_RUNTIME_PREFIX +
+ "sorter.class";
+ public static final String TEZ_RUNTIME_SORTER_CLASS_DEFAULT = SorterImpl.PIPELINED.name();
+
+ public static final String TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS = TEZ_RUNTIME_PREFIX +
+ "pipelined.sorter.sort.threads";
+ public static final int TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS_DEFAULT = 2;
/**
* Size of the buffer to use if not writing directly to disk.
@@ -321,7 +337,7 @@ public class TezRuntimeConfiguration {
tezRuntimeKeys.add(TEZ_RUNTIME_IO_SORT_MB);
tezRuntimeKeys.add(TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES);
tezRuntimeKeys.add(TEZ_RUNTIME_COMBINE_MIN_SPILLS);
- tezRuntimeKeys.add(TEZ_RUNTIME_SORT_THREADS);
+ tezRuntimeKeys.add(TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS);
tezRuntimeKeys.add(TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB);
tezRuntimeKeys.add(TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES);
tezRuntimeKeys.add(TEZ_RUNTIME_PARTITIONER_CLASS);
@@ -357,6 +373,7 @@ public class TezRuntimeConfiguration {
tezRuntimeKeys.add(TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH);
tezRuntimeKeys.add(TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH);
tezRuntimeKeys.add(TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT);
+ tezRuntimeKeys.add(TEZ_RUNTIME_SORTER_CLASS);
defaultConf.addResource("core-default.xml");
defaultConf.addResource("core-site.xml");
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
index 0b959cd..65606bf 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
@@ -177,8 +177,8 @@ public class PipelinedSorter extends ExternalSorter {
merger = new SpanMerger(); // SpanIterators are comparable
final int sortThreads =
this.conf.getInt(
- TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS,
- TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS_DEFAULT);
+ TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS,
+ TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS_DEFAULT);
sortmaster = Executors.newFixedThreadPool(sortThreads,
new ThreadFactoryBuilder().setDaemon(true)
.setNameFormat("Sorter [" + TezUtilsInternal
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
index 5437620..3281617 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
@@ -51,6 +51,19 @@ import org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput;
public class OrderedPartitionedKVOutputConfig {
/**
+ * Currently supported sorter implementations
+ */
+ public enum SorterImpl {
+ /** Legacy sorter implementation based on Hadoop MapReduce shuffle impl.
+ * Restricted to 2 GB memory limits.
+ */
+ LEGACY,
+ /** Pipeline sorter - a more efficient sorter that supports > 2 GB sort buffers */
+ PIPELINED
+ }
+
+
+ /**
* Configure parameters which are specific to the Output.
*/
@InterfaceAudience.Private
@@ -85,8 +98,6 @@ public class OrderedPartitionedKVOutputConfig {
*/
public T setCombiner(String combinerClassName, @Nullable Map<String, String> combinerConf);
-
-
/**
* Configure the number of threads to be used by the sorter
*
@@ -94,6 +105,15 @@ public class OrderedPartitionedKVOutputConfig {
* @return instance of the current builder
*/
public T setSorterNumThreads(int numThreads);
+
+ /**
+ * Configure which sorter implementation to be used
+ *
+ * @param sorterImpl Use an in-built sorter implementations.
+ * @return instance of the current builder
+ */
+ public T setSorter(SorterImpl sorterImpl);
+
}
@SuppressWarnings("rawtypes")
@@ -133,6 +153,13 @@ public class OrderedPartitionedKVOutputConfig {
}
@Override
+ public SpecificBuilder setSorter(SorterImpl sorterImpl) {
+ builder.setSorter(sorterImpl);
+ return this;
+ }
+
+
+ @Override
public SpecificBuilder<E> setAdditionalConfiguration(String key, String value) {
builder.setAdditionalConfiguration(key, value);
return this;
@@ -294,10 +321,19 @@ public class OrderedPartitionedKVOutputConfig {
@Override
public Builder setSorterNumThreads(int numThreads) {
- this.conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS, numThreads);
+ this.conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS, numThreads);
+ return this;
+ }
+
+ @Override
+ public Builder setSorter(SorterImpl sorterImpl) {
+ Preconditions.checkNotNull(sorterImpl, "Sorter cannot be null");
+ this.conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS,
+ sorterImpl.name());
return this;
}
+
@Override
public Builder setAdditionalConfiguration(String key, String value) {
Preconditions.checkNotNull(key, "Key cannot be null");
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
index b03c674..df6daf2 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
@@ -18,15 +18,18 @@
package org.apache.tez.runtime.library.output;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -102,8 +105,17 @@ public class OrderedPartitionedKVOutput extends AbstractLogicalOutput {
public synchronized void start() throws Exception {
if (!isStarted.get()) {
memoryUpdateCallbackHandler.validateUpdateReceived();
- int sortThreads = this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS,
- TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS_DEFAULT);
+ String sorterClass = conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS,
+ TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS_DEFAULT).toUpperCase(Locale.ENGLISH);
+ SorterImpl sorterImpl = null;
+ try {
+ sorterImpl = SorterImpl.valueOf(sorterClass);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Invalid sorter class specified in config"
+ + ", propertyName=" + TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS
+ + ", value=" + sorterClass
+ + ", validValues=" + Arrays.asList(SorterImpl.values()));
+ }
finalMergeEnabled = conf.getBoolean(
TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT,
@@ -121,17 +133,22 @@ public class OrderedPartitionedKVOutput extends AbstractLogicalOutput {
conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
}
- //TODO: Enable it for pipelinedsorter only and not for DefaultSorter
- Preconditions.checkArgument((sortThreads > 1), TezRuntimeConfiguration
- .TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED + " works with PipelinedSorter.");
+ Preconditions.checkArgument(sorterImpl.equals(SorterImpl.PIPELINED),
+ TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED
+ + "only works with PipelinedSorter.");
}
- if (sortThreads > 1) {
+ if (sorterImpl.equals(SorterImpl.PIPELINED)) {
sorter = new PipelinedSorter(getContext(), conf, getNumPhysicalOutputs(),
memoryUpdateCallbackHandler.getMemoryAssigned());
- } else {
+ } else if (sorterImpl.equals(SorterImpl.LEGACY)) {
sorter = new DefaultSorter(getContext(), conf, getNumPhysicalOutputs(),
memoryUpdateCallbackHandler.getMemoryAssigned());
+ } else {
+ throw new UnsupportedOperationException("Unsupported sorter class specified in config"
+ + ", propertyName=" + TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS
+ + ", value=" + sorterClass
+ + ", validValues=" + Arrays.asList(SorterImpl.values()));
}
isStarted.set(true);
@@ -202,7 +219,7 @@ public class OrderedPartitionedKVOutput extends AbstractLogicalOutput {
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINE_MIN_SPILLS);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS);
@@ -219,6 +236,7 @@ public class OrderedPartitionedKVOutput extends AbstractLogicalOutput {
confKeys.add(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
confKeys.add(TezConfiguration.TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH);
confKeys.add(TezConfiguration.TEZ_COUNTERS_MAX_GROUPS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS);
}
// TODO Maybe add helper methods to extract keys
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/api/TestTezRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/api/TestTezRuntimeConfiguration.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/api/TestTezRuntimeConfiguration.java
index fcaebd9..8e97b12 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/api/TestTezRuntimeConfiguration.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/api/TestTezRuntimeConfiguration.java
@@ -27,9 +27,9 @@ import java.lang.reflect.Field;
import java.util.HashSet;
import java.util.Set;
-import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.junit.Test;
+
public class TestTezRuntimeConfiguration {
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java
index f68032f..2e264f6 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java
@@ -103,7 +103,6 @@ public class TestShuffleUtils {
public void setup() throws Exception {
outputContext = createTezOutputContext();
conf = new Configuration();
- conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS, 1); // DefaultSorter
conf.set("fs.defaultFS", "file:///");
localFs = FileSystem.getLocal(conf);
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
index 073d956..6e56567 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
@@ -19,6 +19,7 @@ import org.apache.tez.runtime.api.OutputContext;
import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl;
import org.apache.tez.runtime.library.partitioner.HashPartitioner;
import org.junit.After;
import org.junit.Assert;
@@ -89,7 +90,7 @@ public class TestPipelinedSorter {
this.outputContext = createMockOutputContext(counters, appId, uniqueId);
//To enable PipelinedSorter, set 2 threads
- conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS, 2);
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, SorterImpl.PIPELINED.name());
conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName());
conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, Text.class.getName());
conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
index 408ec3b..70dce13 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
@@ -54,6 +54,7 @@ import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler;
import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils;
import org.apache.tez.runtime.library.common.sort.impl.ExternalSorter;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl;
import org.apache.tez.runtime.library.partitioner.HashPartitioner;
import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads;
import org.junit.After;
@@ -76,7 +77,7 @@ public class TestDefaultSorter {
@Before
public void setup() throws IOException {
conf = new Configuration();
- conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS, 1); // DefaultSorter
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, SorterImpl.LEGACY.name()); // DefaultSorter
conf.set("fs.defaultFS", "file:///");
localFs = FileSystem.getLocal(conf);
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
index ac31cf5..891e807 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
@@ -31,6 +31,7 @@ import java.util.Map;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
+
import org.junit.Test;
public class TestOrderedPartitionedKVOutputConfig {
http://git-wip-us.apache.org/repos/asf/tez/blob/db0a50c5/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
index 8e43f21..30f78fe 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
@@ -39,10 +39,12 @@ import org.apache.tez.runtime.library.api.KeyValuesWriter;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler;
import org.apache.tez.runtime.library.common.sort.impl.dflt.DefaultSorter;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl;
import org.apache.tez.runtime.library.partitioner.HashPartitioner;
import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils;
import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -84,6 +86,7 @@ public class TestOnFileSortedOutput {
//no of outputs
private int partitions;
//For sorter (pipelined / Default)
+ private SorterImpl sorterImpl;
private int sorterThreads;
final AtomicLong outputSize = new AtomicLong();
@@ -98,14 +101,16 @@ public class TestOnFileSortedOutput {
* Constructor
*
* @param sendEmptyPartitionViaEvent
- * @param threads number of threads needed for sorter (pipelinedsorter or default sorter)
+ * @param sorterImpl Which sorter impl ( pipeline/legacy )
+ * @param sorterThreads number of threads needed for sorter (required only for pipelined sorter)
* @param emptyPartitionIdx for which data should not be generated
*/
- public TestOnFileSortedOutput(boolean sendEmptyPartitionViaEvent, int threads,
- int emptyPartitionIdx) throws IOException {
+ public TestOnFileSortedOutput(boolean sendEmptyPartitionViaEvent, SorterImpl sorterImpl,
+ int sorterThreads, int emptyPartitionIdx) throws IOException {
this.sendEmptyPartitionViaEvent = sendEmptyPartitionViaEvent;
this.emptyPartitionIdx = emptyPartitionIdx;
- this.sorterThreads = threads;
+ this.sorterImpl = sorterImpl;
+ this.sorterThreads = sorterThreads;
conf = new Configuration();
@@ -117,7 +122,8 @@ public class TestOnFileSortedOutput {
@Before
public void setup() throws Exception {
- conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS, sorterThreads);
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, sorterImpl.name());
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS, sorterThreads);
conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 5);
conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName());
@@ -138,20 +144,20 @@ public class TestOnFileSortedOutput {
fs.delete(workingDir, true);
}
- @Parameterized.Parameters(name = "test[{0}, {1}, {2}]")
+ @Parameterized.Parameters(name = "test[{0}, {1}, {2}, {3}]")
public static Collection<Object[]> getParameters() {
Collection<Object[]> parameters = new ArrayList<Object[]>();
//empty_partition_via_events_enabled, noOfSortThreads, partitionToBeEmpty
- parameters.add(new Object[] { false, 1, -1 });
- parameters.add(new Object[] { false, 1, 0 });
- parameters.add(new Object[] { true, 1, -1 });
- parameters.add(new Object[] { true, 1, 0 });
+ parameters.add(new Object[] { false, SorterImpl.LEGACY, 1, -1 });
+ parameters.add(new Object[] { false, SorterImpl.LEGACY, 1, 0 });
+ parameters.add(new Object[] { true, SorterImpl.LEGACY, 1, -1 });
+ parameters.add(new Object[] { true, SorterImpl.LEGACY, 1, 0 });
//Pipelined sorter
- parameters.add(new Object[] { false, 2, -1 });
- parameters.add(new Object[] { false, 2, 0 });
- parameters.add(new Object[] { true, 2, -1 });
- parameters.add(new Object[] { true, 2, 0 });
+ parameters.add(new Object[] { false, SorterImpl.PIPELINED, 2, -1 });
+ parameters.add(new Object[] { false, SorterImpl.PIPELINED, 2, 0 });
+ parameters.add(new Object[] { true, SorterImpl.PIPELINED, 2, -1 });
+ parameters.add(new Object[] { true, SorterImpl.PIPELINED, 2, 0 });
return parameters;
}
@@ -167,11 +173,10 @@ public class TestOnFileSortedOutput {
writer = sortedOutput.getWriter();
}
- @Test (timeout = 5000)
- public void testPipelinedShuffle() throws Exception {
+ private void _testPipelinedShuffle(String sorterName) throws Exception {
conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 3);
- conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS, 2);
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, sorterName);
conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, true);
OutputContext context = createTezOutputContext();
@@ -184,14 +189,18 @@ public class TestOnFileSortedOutput {
assertFalse(sortedOutput.finalMergeEnabled);
assertTrue(sortedOutput.pipelinedShuffle);
+ }
+ @Test (timeout = 5000)
+ public void testPipelinedShuffle() throws Exception {
+ _testPipelinedShuffle(SorterImpl.PIPELINED.name());
}
@Test (timeout = 5000)
public void testPipelinedShuffleWithFinalMerge() throws Exception {
conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 3);
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, SorterImpl.PIPELINED.name());
- conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS, 2);
//wrong setting for final merge enable in output
conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);
conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, true);
@@ -209,8 +218,8 @@ public class TestOnFileSortedOutput {
@Test
public void testPipelinedSettingsWithDefaultSorter() throws Exception {
conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 3);
- //negative. with sort threads-1
- conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS, 1);
+ //negative. with default sorter
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, SorterImpl.LEGACY.name());
conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, true);
@@ -376,4 +385,20 @@ public class TestOnFileSortedOutput {
return context;
}
+ @Test(timeout=5000)
+ public void testInvalidSorter() throws Exception {
+ try {
+ _testPipelinedShuffle("Foo");
+ Assert.fail("Expected start to fail due to invalid sorter");
+ } catch (IllegalArgumentException e) {
+ // Expected
+ }
+ }
+
+ @Test(timeout=5000)
+ public void testLowerCaseNamedSorter() throws Exception {
+ _testPipelinedShuffle("Pipelined");
+ }
+
+
}
[38/50] [abbrv] tez git commit: TEZ-2123. Fix component managers to
use pluggable components. Enable hybrid mode. (sseth)
Posted by ss...@apache.org.
TEZ-2123. Fix component managers to use pluggable components. Enable
hybrid mode. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2c5faca3
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2c5faca3
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2c5faca3
Branch: refs/heads/TEZ-2003
Commit: 2c5faca3082bf78719456e58f115c41049d907d8
Parents: 0ae9d4a
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Feb 20 11:59:03 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:46:38 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 4 +-
.../apache/tez/dag/app/TaskAttemptListener.java | 12 +-
.../dag/app/TaskAttemptListenerImpTezDag.java | 31 ++--
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 4 +-
.../TezRootInputInitializerContextImpl.java | 2 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 2 +-
.../tez/dag/app/dag/impl/VertexManager.java | 2 +-
.../app/launcher/ContainerLauncherRouter.java | 2 +-
.../app/launcher/LocalContainerLauncher.java | 10 +-
.../rm/AMSchedulerEventDeallocateContainer.java | 7 +-
.../rm/AMSchedulerEventNodeBlacklistUpdate.java | 8 +-
.../tez/dag/app/rm/AMSchedulerEventTAEnded.java | 10 +-
.../dag/app/rm/LocalTaskSchedulerService.java | 19 ++-
.../tez/dag/app/rm/NMCommunicatorEvent.java | 12 +-
.../rm/NMCommunicatorLaunchRequestEvent.java | 11 +-
.../app/rm/NMCommunicatorStopRequestEvent.java | 4 +-
.../dag/app/rm/TaskSchedulerEventHandler.java | 151 ++++++++++++-----
.../tez/dag/app/rm/container/AMContainer.java | 3 +
.../AMContainerEventLaunchRequest.java | 15 +-
.../dag/app/rm/container/AMContainerImpl.java | 39 +++--
.../dag/app/rm/container/AMContainerMap.java | 4 +-
.../apache/tez/dag/app/rm/node/AMNodeImpl.java | 6 +-
.../apache/tez/dag/app/MockDAGAppMaster.java | 2 +-
.../app/TestTaskAttemptListenerImplTezDag.java | 22 +--
.../tez/dag/app/dag/impl/TestTaskAttempt.java | 69 ++++----
.../tez/dag/app/dag/impl/TestVertexImpl.java | 8 +-
.../tez/dag/app/rm/TestContainerReuse.java | 34 ++--
.../tez/dag/app/rm/TestLocalTaskScheduler.java | 2 +-
.../app/rm/TestLocalTaskSchedulerService.java | 18 ++-
.../app/rm/TestTaskSchedulerEventHandler.java | 11 +-
.../dag/app/rm/TestTaskSchedulerHelpers.java | 2 +-
.../dag/app/rm/container/TestAMContainer.java | 108 +++++++------
.../app/rm/container/TestAMContainerMap.java | 6 +-
.../org/apache/tez/examples/JoinValidate.java | 30 +++-
.../TezTestServiceContainerLauncher.java | 5 +-
.../rm/TezTestServiceTaskSchedulerService.java | 100 ++----------
.../tez/examples/JoinValidateConfigured.java | 53 ++++++
.../tez/tests/TestExternalTezServices.java | 160 ++++++++++++++-----
39 files changed, 630 insertions(+), 359 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 4bfe08f..1a2264c 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -4,5 +4,6 @@ ALL CHANGES:
TEZ-2090. Add tests for jobs running in external services.
TEZ-2117. Add a manager for ContainerLaunchers running in the AM.
TEZ-2122. Setup pluggable components at AM/Vertex level.
+ TEZ-2123. Fix component managers to use pluggable components. (Enable hybrid mode)
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index d386e9e..d31cfe3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -501,7 +501,7 @@ public class DAGAppMaster extends AbstractService {
this.taskSchedulerEventHandler = new TaskSchedulerEventHandler(context,
clientRpcServer, dispatcher.getEventHandler(), containerSignatureMatcher, webUIService,
- taskSchedulerClassIdentifiers);
+ taskSchedulerClassIdentifiers, isLocal);
addIfService(taskSchedulerEventHandler, true);
if (enableWebUIService()) {
@@ -2246,6 +2246,7 @@ public class DAGAppMaster extends AbstractService {
// Tez default classnames are populated as TezConfiguration.TEZ_AM_SERVICE_PLUGINS_DEFAULT
private String[] parsePlugins(BiMap<String, Integer> pluginMap, String[] pluginStrings,
String context) {
+ // TODO TEZ-2003 Duplicate error checking - ideally in the client itself. Depends on the final API.
Preconditions.checkState(pluginStrings != null && pluginStrings.length > 0,
"Plugin strings should not be null or empty: " + context);
@@ -2283,6 +2284,7 @@ public class DAGAppMaster extends AbstractService {
}
pluginMap.put(identifierString, index);
classNames[index] = className;
+ index++;
}
return classNames;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
index 9caa7cf..e4dad27 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
@@ -22,6 +22,7 @@ import java.net.InetSocketAddress;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.app.rm.container.AMContainerTask;
import org.apache.tez.dag.records.TezTaskAttemptID;
/**
@@ -29,18 +30,17 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
*/
public interface TaskAttemptListener {
- InetSocketAddress getAddress();
+ void registerRunningContainer(ContainerId containerId, int taskCommId);
- void registerRunningContainer(ContainerId containerId);
-
- void registerTaskAttempt(AMContainerTask amContainerTask, ContainerId containerId);
+ void registerTaskAttempt(AMContainerTask amContainerTask, ContainerId containerId, int taskCommId);
- void unregisterRunningContainer(ContainerId containerId);
+ void unregisterRunningContainer(ContainerId containerId, int taskCommId);
- void unregisterTaskAttempt(TezTaskAttemptID attemptID);
+ void unregisterTaskAttempt(TezTaskAttemptID attemptID, int taskCommId);
void dagComplete(DAG dag);
void dagSubmitted();
+ TaskCommunicator getTaskCommunicator(int taskCommIndex);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 94e4f71..adb21e7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -251,11 +251,6 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
return task.canCommit(taskAttemptId);
}
- @Override
- public InetSocketAddress getAddress() {
- return taskCommunicators[0].getAddress();
- }
-
// The TaskAttemptListener register / unregister methods in this class are not thread safe.
// The Tez framework should not invoke these methods from multiple threads.
@Override
@@ -275,7 +270,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
@Override
- public void registerRunningContainer(ContainerId containerId) {
+ public void registerRunningContainer(ContainerId containerId, int taskCommId) {
if (LOG.isDebugEnabled()) {
LOG.debug("ContainerId: " + containerId + " registered with TaskAttemptListener");
}
@@ -285,11 +280,12 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
"Multiple registrations for containerId: " + containerId);
}
NodeId nodeId = context.getAllContainers().get(containerId).getContainer().getNodeId();
- taskCommunicators[0].registerRunningContainer(containerId, nodeId.getHost(), nodeId.getPort());
+ taskCommunicators[taskCommId].registerRunningContainer(containerId, nodeId.getHost(),
+ nodeId.getPort());
}
@Override
- public void unregisterRunningContainer(ContainerId containerId) {
+ public void unregisterRunningContainer(ContainerId containerId, int taskCommId) {
if (LOG.isDebugEnabled()) {
LOG.debug("Unregistering Container from TaskAttemptListener: " + containerId);
}
@@ -297,12 +293,12 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
if (containerInfo.taskAttemptId != null) {
registeredAttempts.remove(containerInfo.taskAttemptId);
}
- taskCommunicators[0].registerContainerEnd(containerId);
+ taskCommunicators[taskCommId].registerContainerEnd(containerId);
}
@Override
public void registerTaskAttempt(AMContainerTask amContainerTask,
- ContainerId containerId) {
+ ContainerId containerId, int taskCommId) {
ContainerInfo containerInfo = registeredContainers.get(containerId);
if (containerInfo == null) {
throw new TezUncheckedException("Registering task attempt: "
@@ -332,13 +328,13 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
+ amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+ " when already assigned to: " + containerIdFromMap);
}
- taskCommunicators[0].registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
+ taskCommunicators[taskCommId].registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
amContainerTask.getAdditionalResources(), amContainerTask.getCredentials(),
amContainerTask.haveCredentialsChanged());
}
@Override
- public void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
+ public void unregisterTaskAttempt(TezTaskAttemptID attemptId, int taskCommId) {
ContainerId containerId = registeredAttempts.remove(attemptId);
if (containerId == null) {
LOG.warn("Unregister task attempt: " + attemptId + " from unknown container");
@@ -352,7 +348,12 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
// Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
registeredContainers.put(containerId, NULL_CONTAINER_INFO);
- taskCommunicators[0].unregisterRunningTaskAttempt(attemptId);
+ taskCommunicators[taskCommId].unregisterRunningTaskAttempt(attemptId);
+ }
+
+ @Override
+ public TaskCommunicator getTaskCommunicator(int taskCommIndex) {
+ return taskCommunicators[taskCommIndex];
}
private void pingContainerHeartbeatHandler(ContainerId containerId) {
@@ -368,8 +369,4 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
+ ", ContainerId not known for this attempt");
}
}
-
- public TaskCommunicator getTaskCommunicator() {
- return taskCommunicators[0];
- }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index b5a6c9c..794ef83 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -1214,7 +1214,7 @@ public class TaskAttemptImpl implements TaskAttempt,
// Inform the scheduler
if (sendSchedulerEvent()) {
ta.sendEvent(new AMSchedulerEventTAEnded(ta, ta.containerId, helper
- .getTaskAttemptState()));
+ .getTaskAttemptState(), ta.getVertex().getTaskSchedulerIdentifier()));
}
}
}
@@ -1296,7 +1296,7 @@ public class TaskAttemptImpl implements TaskAttempt,
// Inform the Scheduler.
ta.sendEvent(new AMSchedulerEventTAEnded(ta, ta.containerId,
- TaskAttemptState.SUCCEEDED));
+ TaskAttemptState.SUCCEEDED, ta.getVertex().getTaskSchedulerIdentifier()));
// Inform the task.
ta.sendEvent(new TaskEventTAUpdate(ta.attemptId,
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java
index d4ef4d5..4ca4024 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java
@@ -96,7 +96,7 @@ public class TezRootInputInitializerContextImpl implements
@Override
public Resource getTotalAvailableResource() {
- return appContext.getTaskScheduler().getTotalResources();
+ return appContext.getTaskScheduler().getTotalResources(vertex.getTaskSchedulerIdentifier());
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index bce4127..58607e6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -4326,7 +4326,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
eventHandler, getTotalTasks(),
appContext.getTaskScheduler().getNumClusterNodes(),
getTaskResource(),
- appContext.getTaskScheduler().getTotalResources());
+ appContext.getTaskScheduler().getTotalResources(taskSchedulerIdentifier));
List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>>
inputList = Lists.newArrayListWithCapacity(inputsWithInitializers.size());
for (String inputName : inputsWithInitializers) {
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
index 1ed42fc..94f5efc 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
@@ -284,7 +284,7 @@ public class VertexManager {
@Override
public synchronized Resource getTotalAvailableResource() {
checkAndThrowIfDone();
- return appContext.getTaskScheduler().getTotalResources();
+ return appContext.getTaskScheduler().getTotalResources(managedVertex.getTaskSchedulerIdentifier());
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
index 621e4a8..4f9b5bf 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
@@ -124,6 +124,6 @@ public class ContainerLauncherRouter extends AbstractService
@Override
public void handle(NMCommunicatorEvent event) {
- containerLaunchers[0].handle(event);
+ containerLaunchers[event.getLauncherId()].handle(event);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index e9ba9d7..9a38732 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -59,7 +59,6 @@ import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
-import org.apache.tez.dag.app.TaskAttemptListenerImpTezDag;
import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
@@ -88,9 +87,9 @@ public class LocalContainerLauncher extends AbstractService implements
private static final Logger LOG = LoggerFactory.getLogger(LocalContainerLauncher.class);
private final AppContext context;
- private final TezTaskUmbilicalProtocol taskUmbilicalProtocol;
private final AtomicBoolean serviceStopped = new AtomicBoolean(false);
private final String workingDirectory;
+ private final TaskAttemptListener tal;
private final Map<String, String> localEnv = new HashMap<String, String>();
private final ExecutionContext executionContext;
private int numExecutors;
@@ -116,9 +115,8 @@ public class LocalContainerLauncher extends AbstractService implements
String workingDirectory) throws UnknownHostException {
super(LocalContainerLauncher.class.getName());
this.context = context;
- TaskAttemptListenerImpTezDag taListener = (TaskAttemptListenerImpTezDag)taskAttemptListener;
- TezTaskCommunicatorImpl taskComm = (TezTaskCommunicatorImpl) taListener.getTaskCommunicator();
- this.taskUmbilicalProtocol = taskComm.getUmbilical();
+ this.tal = taskAttemptListener;
+
this.workingDirectory = workingDirectory;
AuxiliaryServiceHelper.setServiceDataIntoEnv(
ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, ByteBuffer.allocate(4).putInt(0), localEnv);
@@ -219,7 +217,7 @@ public class LocalContainerLauncher extends AbstractService implements
tezChild =
createTezChild(context.getAMConf(), event.getContainerId(), tokenIdentifier,
context.getApplicationAttemptId().getAttemptId(), context.getLocalDirs(),
- taskUmbilicalProtocol,
+ ((TezTaskCommunicatorImpl)tal.getTaskCommunicator(event.getTaskCommId())).getUmbilical(),
TezCommonUtils.parseCredentialsBytes(event.getContainerLaunchContext().getTokens().array()));
} catch (InterruptedException e) {
handleLaunchFailed(e, event.getContainerId());
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java
index 1b51920..5270aa2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java
@@ -23,15 +23,20 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
public class AMSchedulerEventDeallocateContainer extends AMSchedulerEvent {
private final ContainerId containerId;
+ private final int schedulerId;
- public AMSchedulerEventDeallocateContainer(ContainerId containerId) {
+ public AMSchedulerEventDeallocateContainer(ContainerId containerId, int schedulerId) {
super(AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
this.containerId = containerId;
+ this.schedulerId = schedulerId;
}
public ContainerId getContainerId() {
return this.containerId;
}
+ public int getSchedulerId() {
+ return schedulerId;
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventNodeBlacklistUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventNodeBlacklistUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventNodeBlacklistUpdate.java
index ed7ebc3..679705a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventNodeBlacklistUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventNodeBlacklistUpdate.java
@@ -23,14 +23,20 @@ import org.apache.hadoop.yarn.api.records.NodeId;
public class AMSchedulerEventNodeBlacklistUpdate extends AMSchedulerEvent {
private final NodeId nodeId;
+ private final int schedulerId;
- public AMSchedulerEventNodeBlacklistUpdate(NodeId nodeId, boolean add) {
+ public AMSchedulerEventNodeBlacklistUpdate(NodeId nodeId, boolean add, int schedulerId) {
super((add ? AMSchedulerEventType.S_NODE_BLACKLISTED
: AMSchedulerEventType.S_NODE_UNBLACKLISTED));
this.nodeId = nodeId;
+ this.schedulerId = schedulerId;
}
public NodeId getNodeId() {
return this.nodeId;
}
+
+ public int getSchedulerId() {
+ return schedulerId;
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
index 90e76b7..2ace642 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
@@ -26,14 +26,16 @@ public class AMSchedulerEventTAEnded extends AMSchedulerEvent {
private final TaskAttempt attempt;
private final ContainerId containerId;
- private TaskAttemptState state;
+ private final TaskAttemptState state;
+ private final int schedulerId;
public AMSchedulerEventTAEnded(TaskAttempt attempt, ContainerId containerId,
- TaskAttemptState state) {
+ TaskAttemptState state, int schedulerId) {
super(AMSchedulerEventType.S_TA_ENDED);
this.attempt = attempt;
this.containerId = containerId;
this.state = state;
+ this.schedulerId = schedulerId;
}
public TezTaskAttemptID getAttemptID() {
@@ -51,4 +53,8 @@ public class AMSchedulerEventTAEnded extends AMSchedulerEvent {
public ContainerId getUsedContainerId() {
return this.containerId;
}
+
+ public int getSchedulerId() {
+ return schedulerId;
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
index 51d8b9d..72a074f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
@@ -34,6 +34,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -63,10 +64,11 @@ public class LocalTaskSchedulerService extends TaskSchedulerService {
final int appHostPort;
final String appTrackingUrl;
final AppContext appContext;
+ final long customContainerAppId;
public LocalTaskSchedulerService(TaskSchedulerAppCallback appClient,
ContainerSignatureMatcher containerSignatureMatcher, String appHostName,
- int appHostPort, String appTrackingUrl, AppContext appContext) {
+ int appHostPort, String appTrackingUrl, long customContainerAppId, AppContext appContext) {
super(LocalTaskSchedulerService.class.getName());
this.realAppClient = appClient;
this.appCallbackExecutor = createAppCallbackExecutorService();
@@ -78,6 +80,7 @@ public class LocalTaskSchedulerService extends TaskSchedulerService {
this.appContext = appContext;
taskRequestQueue = new PriorityBlockingQueue<TaskRequest>();
taskAllocations = new LinkedHashMap<Object, Container>();
+ this.customContainerAppId = customContainerAppId;
}
private ExecutorService createAppCallbackExecutorService() {
@@ -164,7 +167,7 @@ public class LocalTaskSchedulerService extends TaskSchedulerService {
protected AsyncDelegateRequestHandler createRequestHandler(Configuration conf) {
return new AsyncDelegateRequestHandler(taskRequestQueue,
- new LocalContainerFactory(appContext),
+ new LocalContainerFactory(appContext, customContainerAppId),
taskAllocations,
appClientDelegate,
conf);
@@ -195,17 +198,19 @@ public class LocalTaskSchedulerService extends TaskSchedulerService {
}
static class LocalContainerFactory {
- final AppContext appContext;
AtomicInteger nextId;
+ final ApplicationAttemptId customAppAttemptId;
- public LocalContainerFactory(AppContext appContext) {
- this.appContext = appContext;
+ public LocalContainerFactory(AppContext appContext, long appIdLong) {
this.nextId = new AtomicInteger(1);
+ ApplicationId appId = ApplicationId
+ .newInstance(appIdLong, appContext.getApplicationAttemptId().getApplicationId().getId());
+ this.customAppAttemptId = ApplicationAttemptId
+ .newInstance(appId, appContext.getApplicationAttemptId().getAttemptId());
}
public Container createContainer(Resource capability, Priority priority) {
- ApplicationAttemptId appAttemptId = appContext.getApplicationAttemptId();
- ContainerId containerId = ContainerId.newInstance(appAttemptId, nextId.getAndIncrement());
+ ContainerId containerId = ContainerId.newInstance(customAppAttemptId, nextId.getAndIncrement());
NodeId nodeId = NodeId.newInstance("127.0.0.1", 0);
String nodeHttpAddress = "127.0.0.1:0";
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
index 8bdeb28..f86894f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
@@ -28,13 +28,15 @@ public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType>
private final ContainerId containerId;
private final NodeId nodeId;
private final Token containerToken;
+ private final int launcherId;
public NMCommunicatorEvent(ContainerId containerId, NodeId nodeId,
- Token containerToken, NMCommunicatorEventType type) {
+ Token containerToken, NMCommunicatorEventType type, int launcherId) {
super(type);
this.containerId = containerId;
this.nodeId = nodeId;
this.containerToken = containerToken;
+ this.launcherId = launcherId;
}
public ContainerId getContainerId() {
@@ -48,10 +50,14 @@ public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType>
public Token getContainerToken() {
return this.containerToken;
}
-
+
+ public int getLauncherId() {
+ return launcherId;
+ }
+
public String toSrting() {
return super.toString() + " for container " + containerId + ", nodeId: "
- + nodeId;
+ + nodeId + ", launcherId: " + launcherId;
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java
index c3b12c0..a38345c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java
@@ -25,13 +25,16 @@ public class NMCommunicatorLaunchRequestEvent extends NMCommunicatorEvent {
private final ContainerLaunchContext clc;
private final Container container;
+ // The task communicator index for the specific container being launched.
+ private final int taskCommId;
public NMCommunicatorLaunchRequestEvent(ContainerLaunchContext clc,
- Container container) {
+ Container container, int launcherId, int taskCommId) {
super(container.getId(), container.getNodeId(), container
- .getContainerToken(), NMCommunicatorEventType.CONTAINER_LAUNCH_REQUEST);
+ .getContainerToken(), NMCommunicatorEventType.CONTAINER_LAUNCH_REQUEST, launcherId);
this.clc = clc;
this.container = container;
+ this.taskCommId = taskCommId;
}
public ContainerLaunchContext getContainerLaunchContext() {
@@ -42,6 +45,10 @@ public class NMCommunicatorLaunchRequestEvent extends NMCommunicatorEvent {
return container;
}
+ public int getTaskCommId() {
+ return taskCommId;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
index 277d1e7..c9b5c44 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
@@ -25,9 +25,9 @@ import org.apache.hadoop.yarn.api.records.Token;
public class NMCommunicatorStopRequestEvent extends NMCommunicatorEvent {
public NMCommunicatorStopRequestEvent(ContainerId containerId, NodeId nodeId,
- Token containerToken) {
+ Token containerToken, int launcherId) {
super(containerId, nodeId, containerToken,
- NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+ NMCommunicatorEventType.CONTAINER_STOP_REQUEST, launcherId);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 72389e7..5a0ace8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -108,9 +108,22 @@ public class TaskSchedulerEventHandler extends AbstractService
private final String[] taskSchedulerClasses;
protected final TaskSchedulerService []taskSchedulers;
+ private final boolean isPureLocalMode;
+ // If running in non local-only mode, the YARN task scheduler will always run to take care of
+ // registration with YARN and heartbeats to YARN.
+ // Splitting registration and heartbeats is not straigh-forward due to the taskScheduler being
+ // tied to a ContainerRequestType.
+ private final int yarnTaskSchedulerIndex;
+ // Custom AppIds to avoid container conflicts if there's multiple sources
+ private final long SCHEDULER_APP_ID_BASE = 111101111;
+ private final long SCHEDULER_APP_ID_INCREMENT = 111111111;
+
BlockingQueue<AMSchedulerEvent> eventQueue
= new LinkedBlockingQueue<AMSchedulerEvent>();
+ // Not tracking container / task to schedulerId. Instead relying on everything flowing through
+ // the system and being propagated back via events.
+
/**
*
* @param appContext
@@ -125,7 +138,7 @@ public class TaskSchedulerEventHandler extends AbstractService
public TaskSchedulerEventHandler(AppContext appContext,
DAGClientServer clientService, EventHandler eventHandler,
ContainerSignatureMatcher containerSignatureMatcher, WebUIService webUI,
- String [] schedulerClasses) {
+ String [] schedulerClasses, boolean isPureLocalMode) {
super(TaskSchedulerEventHandler.class.getName());
this.appContext = appContext;
this.eventHandler = eventHandler;
@@ -133,13 +146,39 @@ public class TaskSchedulerEventHandler extends AbstractService
this.containerSignatureMatcher = containerSignatureMatcher;
this.webUI = webUI;
this.historyUrl = getHistoryUrl();
+ this.isPureLocalMode = isPureLocalMode;
if (this.webUI != null) {
this.webUI.setHistoryUrl(this.historyUrl);
}
- if (schedulerClasses == null || schedulerClasses.length == 0) {
- this.taskSchedulerClasses = new String[] {TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT};
+
+ // Override everything for pure local mode
+ if (isPureLocalMode) {
+ this.taskSchedulerClasses = new String[] {TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT};
+ this.yarnTaskSchedulerIndex = -1;
} else {
- this.taskSchedulerClasses = schedulerClasses;
+ if (schedulerClasses == null || schedulerClasses.length ==0) {
+ this.taskSchedulerClasses = new String[] {TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT};
+ this.yarnTaskSchedulerIndex = 0;
+ } else {
+ // Ensure the YarnScheduler will be setup and note it's index. This will be responsible for heartbeats and YARN registration.
+ int foundYarnTaskSchedulerIndex = -1;
+ for (int i = 0 ; i < schedulerClasses.length ; i++) {
+ if (schedulerClasses[i].equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT)) {
+ foundYarnTaskSchedulerIndex = i;
+ break;
+ }
+ }
+ if (foundYarnTaskSchedulerIndex == -1) { // Not found. Add at the end.
+ this.taskSchedulerClasses = new String[schedulerClasses.length+1];
+ foundYarnTaskSchedulerIndex = this.taskSchedulerClasses.length -1;
+ for (int i = 0 ; i < schedulerClasses.length ; i++) { // Copy over the rest.
+ this.taskSchedulerClasses[i] = schedulerClasses[i];
+ }
+ } else {
+ this.taskSchedulerClasses = schedulerClasses;
+ }
+ this.yarnTaskSchedulerIndex = foundYarnTaskSchedulerIndex;
+ }
}
taskSchedulers = new TaskSchedulerService[this.taskSchedulerClasses.length];
}
@@ -157,12 +196,12 @@ public class TaskSchedulerEventHandler extends AbstractService
return cachedNodeCount;
}
- public Resource getAvailableResources() {
- return taskSchedulers[0].getAvailableResources();
+ public Resource getAvailableResources(int schedulerId) {
+ return taskSchedulers[schedulerId].getAvailableResources();
}
- public Resource getTotalResources() {
- return taskSchedulers[0].getTotalResources();
+ public Resource getTotalResources(int schedulerId) {
+ return taskSchedulers[schedulerId].getTotalResources();
}
public synchronized void handleEvent(AMSchedulerEvent sEvent) {
@@ -176,7 +215,7 @@ public class TaskSchedulerEventHandler extends AbstractService
switch(event.getState()) {
case FAILED:
case KILLED:
- handleTAUnsuccessfulEnd((AMSchedulerEventTAEnded) sEvent);
+ handleTAUnsuccessfulEnd(event);
break;
case SUCCEEDED:
handleTASucceeded(event);
@@ -228,9 +267,9 @@ public class TaskSchedulerEventHandler extends AbstractService
private void handleNodeBlacklistUpdate(AMSchedulerEventNodeBlacklistUpdate event) {
if (event.getType() == AMSchedulerEventType.S_NODE_BLACKLISTED) {
- taskSchedulers[0].blacklistNode(event.getNodeId());
+ taskSchedulers[event.getSchedulerId()].blacklistNode(event.getNodeId());
} else if (event.getType() == AMSchedulerEventType.S_NODE_UNBLACKLISTED) {
- taskSchedulers[0].unblacklistNode(event.getNodeId());
+ taskSchedulers[event.getSchedulerId()].unblacklistNode(event.getNodeId());
} else {
throw new TezUncheckedException("Invalid event type: " + event.getType());
}
@@ -242,14 +281,14 @@ public class TaskSchedulerEventHandler extends AbstractService
// TODO what happens to the task that was connected to this container?
// current assumption is that it will eventually call handleTaStopRequest
//TaskAttempt taskAttempt = (TaskAttempt)
- taskSchedulers[0].deallocateContainer(containerId);
+ taskSchedulers[event.getSchedulerId()].deallocateContainer(containerId);
// TODO does this container need to be stopped via C_STOP_REQUEST
sendEvent(new AMContainerEventStopRequest(containerId));
}
private void handleTAUnsuccessfulEnd(AMSchedulerEventTAEnded event) {
TaskAttempt attempt = event.getAttempt();
- boolean wasContainerAllocated = taskSchedulers[0].deallocateTask(attempt, false);
+ boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()].deallocateTask(attempt, false);
// use stored value of container id in case the scheduler has removed this
// assignment because the task has been deallocated earlier.
// retroactive case
@@ -291,7 +330,8 @@ public class TaskSchedulerEventHandler extends AbstractService
event.getAttemptID()));
}
- boolean wasContainerAllocated = taskSchedulers[0].deallocateTask(attempt, true);
+ boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()].deallocateTask(attempt,
+ true);
if (!wasContainerAllocated) {
LOG.error("De-allocated successful task: " + attempt.getID()
+ ", but TaskScheduler reported no container assigned to task");
@@ -316,7 +356,7 @@ public class TaskSchedulerEventHandler extends AbstractService
TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt();
if (affinityAttempt != null) {
Preconditions.checkNotNull(affinityAttempt.getAssignedContainerID(), affinityAttempt.getID());
- taskSchedulers[0].allocateTask(taskAttempt,
+ taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt,
event.getCapability(),
affinityAttempt.getAssignedContainerID(),
Priority.newInstance(event.getPriority()),
@@ -336,7 +376,7 @@ public class TaskSchedulerEventHandler extends AbstractService
}
}
- taskSchedulers[0].allocateTask(taskAttempt,
+ taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt,
event.getCapability(),
hosts,
racks,
@@ -347,7 +387,8 @@ public class TaskSchedulerEventHandler extends AbstractService
private TaskSchedulerService createTaskScheduler(String host, int port, String trackingUrl,
AppContext appContext,
- String schedulerClassName) {
+ String schedulerClassName,
+ long customAppIdIdentifier) {
if (schedulerClassName.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT)) {
LOG.info("Creating TaskScheduler: YarnTaskSchedulerService");
return new YarnTaskSchedulerService(this, this.containerSignatureMatcher,
@@ -355,7 +396,7 @@ public class TaskSchedulerEventHandler extends AbstractService
} else if (schedulerClassName.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT)) {
LOG.info("Creating TaskScheduler: Local TaskScheduler");
return new LocalTaskSchedulerService(this, this.containerSignatureMatcher,
- host, port, trackingUrl, appContext);
+ host, port, trackingUrl, customAppIdIdentifier, appContext);
} else {
LOG.info("Creating custom TaskScheduler: " + schedulerClassName);
// TODO TEZ-2003 Temporary reflection with specific parameters. Remove once there is a clean interface.
@@ -364,9 +405,10 @@ public class TaskSchedulerEventHandler extends AbstractService
try {
Constructor<? extends TaskSchedulerService> ctor = taskSchedulerClazz
.getConstructor(TaskSchedulerAppCallback.class, AppContext.class, String.class,
- int.class, String.class, Configuration.class);
+ int.class, String.class, long.class, Configuration.class);
ctor.setAccessible(true);
- return ctor.newInstance(this, appContext, host, port, trackingUrl, getConfig());
+ return ctor.newInstance(this, appContext, host, port, trackingUrl, customAppIdIdentifier,
+ getConfig());
} catch (NoSuchMethodException e) {
throw new TezUncheckedException(e);
} catch (InvocationTargetException e) {
@@ -381,10 +423,19 @@ public class TaskSchedulerEventHandler extends AbstractService
@VisibleForTesting
protected void instantiateScheduelrs(String host, int port, String trackingUrl, AppContext appContext) {
+ // TODO Add error checking for components being used in the Vertex when running in pure local mode.
// Iterate over the list and create all the taskSchedulers
+ int j = 0;
for (int i = 0; i < taskSchedulerClasses.length; i++) {
+ long customAppIdIdentifier;
+ if (isPureLocalMode || taskSchedulerClasses[i].equals(
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT)) { // Use the app identifier from the appId.
+ customAppIdIdentifier = appContext.getApplicationID().getClusterTimestamp();
+ } else {
+ customAppIdIdentifier = SCHEDULER_APP_ID_BASE + (j++ * SCHEDULER_APP_ID_INCREMENT);
+ }
taskSchedulers[i] = createTaskScheduler(host, port,
- trackingUrl, appContext, taskSchedulerClasses[i]);
+ trackingUrl, appContext, taskSchedulerClasses[i], customAppIdIdentifier);
}
}
@@ -403,12 +454,12 @@ public class TaskSchedulerEventHandler extends AbstractService
for (int i = 0 ; i < taskSchedulers.length ; i++) {
taskSchedulers[i].init(getConfig());
taskSchedulers[i].start();
- }
-
- // TODO TEZ-2118 Start using multiple task schedulers
- if (shouldUnregisterFlag.get()) {
- // Flag may have been set earlier when task scheduler was not initialized
- taskSchedulers[0].setShouldUnregister();
+ if (shouldUnregisterFlag.get()) {
+ // Flag may have been set earlier when task scheduler was not initialized
+ // TODO TEZ-2003 Should setRegister / unregister be part of APIs when not YARN specific ?
+ // External services could need to talk to some other entity.
+ taskSchedulers[i].setShouldUnregister();
+ }
}
this.eventHandlingThread = new Thread("TaskSchedulerEventHandlerThread") {
@@ -457,8 +508,10 @@ public class TaskSchedulerEventHandler extends AbstractService
if (eventHandlingThread != null)
eventHandlingThread.interrupt();
}
- if (taskSchedulers[0] != null) {
- ((AbstractService)taskSchedulers[0]).stop();
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ if (taskSchedulers[i] != null) {
+ taskSchedulers[i].stop();
+ }
}
}
@@ -467,15 +520,18 @@ public class TaskSchedulerEventHandler extends AbstractService
public synchronized void taskAllocated(Object task,
Object appCookie,
Container container) {
+ AMSchedulerEventTALaunchRequest event =
+ (AMSchedulerEventTALaunchRequest) appCookie;
ContainerId containerId = container.getId();
- if (appContext.getAllContainers().addContainerIfNew(container)) {
+ if (appContext.getAllContainers()
+ .addContainerIfNew(container, event.getSchedulerId(), event.getLauncherId(),
+ event.getTaskCommId())) {
appContext.getNodeTracker().nodeSeen(container.getNodeId());
sendEvent(new AMNodeEventContainerAllocated(container
.getNodeId(), container.getId()));
}
- AMSchedulerEventTALaunchRequest event =
- (AMSchedulerEventTALaunchRequest) appCookie;
+
TaskAttempt taskAttempt = event.getTaskAttempt();
// TODO - perhaps check if the task still needs this container
// because the deallocateTask downcall may have raced with the
@@ -484,7 +540,7 @@ public class TaskSchedulerEventHandler extends AbstractService
if (appContext.getAllContainers().get(containerId).getState() == AMContainerState.ALLOCATED) {
sendEvent(new AMContainerEventLaunchRequest(containerId, taskAttempt.getVertexID(),
- event.getContainerContext()));
+ event.getContainerContext(), event.getLauncherId(), event.getTaskCommId()));
}
sendEvent(new DAGEventSchedulerUpdateTAAssigned(taskAttempt, container));
sendEvent(new AMContainerEventAssignTA(containerId, taskAttempt.getID(),
@@ -603,6 +659,9 @@ public class TaskSchedulerEventHandler extends AbstractService
public float getProgress() {
// at this point allocate has been called and so node count must be available
// may change after YARN-1722
+ // This is a heartbeat in from the scheduler into the APP, and is being used to piggy-back and
+ // node updates from the cluster.
+ // TODO Handle this in TEZ-2124. Need a way to know which scheduler is calling in.
int nodeCount = taskSchedulers[0].getClusterNodeCount();
if (nodeCount != cachedNodeCount) {
cachedNodeCount = nodeCount;
@@ -618,7 +677,9 @@ public class TaskSchedulerEventHandler extends AbstractService
}
public void dagCompleted() {
- taskSchedulers[0].dagComplete();
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ taskSchedulers[i].dagComplete();
+ }
}
public void dagSubmitted() {
@@ -628,7 +689,10 @@ public class TaskSchedulerEventHandler extends AbstractService
@Override
public void preemptContainer(ContainerId containerId) {
- taskSchedulers[0].deallocateContainer(containerId);
+ // TODO Why is this making a call back into the scheduler, when the call is originating from there.
+ // An AMContainer instance should already exist if an attempt is being made to preempt it
+ AMContainer amContainer = appContext.getAllContainers().get(containerId);
+ taskSchedulers[amContainer.getTaskSchedulerIdentifier()].deallocateContainer(containerId);
// Inform the Containers about completion.
sendEvent(new AMContainerEventCompleted(containerId, ContainerExitStatus.INVALID,
"Container preempted internally", TaskAttemptTerminationCause.INTERNAL_PREEMPTION));
@@ -637,13 +701,24 @@ public class TaskSchedulerEventHandler extends AbstractService
public void setShouldUnregisterFlag() {
LOG.info("TaskScheduler notified that it should unregister from RM");
this.shouldUnregisterFlag.set(true);
- if (this.taskSchedulers[0] != null) {
- this.taskSchedulers[0].setShouldUnregister();
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ if (this.taskSchedulers[i] != null) {
+ // TODO TEZ-2003 registration required for all schedulers ?
+ this.taskSchedulers[i].setShouldUnregister();
+ }
}
}
public boolean hasUnregistered() {
- return this.taskSchedulers[0].hasUnregistered();
+ boolean result = true;
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ // TODO TEZ-2003 registration required for all schedulers ?
+ result |= this.taskSchedulers[i].hasUnregistered();
+ if (result == false) {
+ return result;
+ }
+ }
+ return result;
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
index 0fc2e12..6616896 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
@@ -34,4 +34,7 @@ public interface AMContainer extends EventHandler<AMContainerEvent>{
public List<TezTaskAttemptID> getAllTaskAttempts();
public TezTaskAttemptID getCurrentTaskAttempt();
+ public int getTaskSchedulerIdentifier();
+ public int getContainerLauncherIdentifier();
+ public int getTaskCommunicatorIdentifier();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
index d973264..92e5817 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
@@ -27,12 +27,17 @@ public class AMContainerEventLaunchRequest extends AMContainerEvent {
private final TezVertexID vertexId;
private final ContainerContext containerContext;
+ private final int launcherId;
+ private final int taskCommId;
public AMContainerEventLaunchRequest(ContainerId containerId,
- TezVertexID vertexId, ContainerContext containerContext) {
+ TezVertexID vertexId, ContainerContext containerContext,
+ int launcherId, int taskCommId) {
super(containerId, AMContainerEventType.C_LAUNCH_REQUEST);
this.vertexId = vertexId;
this.containerContext = containerContext;
+ this.launcherId = launcherId;
+ this.taskCommId = taskCommId;
}
public TezDAGID getDAGId() {
@@ -46,4 +51,12 @@ public class AMContainerEventLaunchRequest extends AMContainerEvent {
public ContainerContext getContainerContext() {
return this.containerContext;
}
+
+ public int getLauncherId() {
+ return launcherId;
+ }
+
+ public int getTaskCommId() {
+ return taskCommId;
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 1acec9c..39df2e8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -81,6 +81,9 @@ public class AMContainerImpl implements AMContainer {
private final TaskAttemptListener taskAttemptListener;
protected final EventHandler eventHandler;
private final ContainerSignatureMatcher signatureMatcher;
+ private final int schedulerId;
+ private final int launcherId;
+ private final int taskCommId;
private final List<TezTaskAttemptID> completedAttempts =
new LinkedList<TezTaskAttemptID>();
@@ -302,7 +305,7 @@ public class AMContainerImpl implements AMContainer {
// additional change - JvmID, YarnChild, etc depend on TaskType.
public AMContainerImpl(Container container, ContainerHeartbeatHandler chh,
TaskAttemptListener tal, ContainerSignatureMatcher signatureMatcher,
- AppContext appContext) {
+ AppContext appContext, int schedulerId, int launcherId, int taskCommId) {
ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
this.readLock = rwLock.readLock();
this.writeLock = rwLock.writeLock();
@@ -314,6 +317,9 @@ public class AMContainerImpl implements AMContainer {
this.containerHeartbeatHandler = chh;
this.taskAttemptListener = tal;
this.failedAssignments = new LinkedList<TezTaskAttemptID>();
+ this.schedulerId = schedulerId;
+ this.launcherId = launcherId;
+ this.taskCommId = taskCommId;
this.stateMachine = stateMachineFactory.make(this);
}
@@ -363,6 +369,21 @@ public class AMContainerImpl implements AMContainer {
}
}
+ @Override
+ public int getTaskSchedulerIdentifier() {
+ return this.schedulerId;
+ }
+
+ @Override
+ public int getContainerLauncherIdentifier() {
+ return this.launcherId;
+ }
+
+ @Override
+ public int getTaskCommunicatorIdentifier() {
+ return this.taskCommId;
+ }
+
public boolean isInErrorState() {
return inError;
}
@@ -432,7 +453,7 @@ public class AMContainerImpl implements AMContainer {
containerContext.getLocalResources(),
containerContext.getEnvironment(),
containerContext.getJavaOpts(),
- container.taskAttemptListener.getAddress(), containerContext.getCredentials(),
+ container.taskAttemptListener.getTaskCommunicator(container.taskCommId).getAddress(), containerContext.getCredentials(),
container.appContext, container.container.getResource(),
container.appContext.getAMConf());
@@ -1014,7 +1035,7 @@ public class AMContainerImpl implements AMContainer {
}
protected void deAllocate() {
- sendEvent(new AMSchedulerEventDeallocateContainer(containerId));
+ sendEvent(new AMSchedulerEventDeallocateContainer(containerId, schedulerId));
}
protected void sendTerminatedToTaskAttempt(
@@ -1044,28 +1065,28 @@ public class AMContainerImpl implements AMContainer {
}
protected void sendStartRequestToNM(ContainerLaunchContext clc) {
- sendEvent(new NMCommunicatorLaunchRequestEvent(clc, container));
+ sendEvent(new NMCommunicatorLaunchRequestEvent(clc, container, launcherId, taskCommId));
}
protected void sendStopRequestToNM() {
sendEvent(new NMCommunicatorStopRequestEvent(containerId,
- container.getNodeId(), container.getContainerToken()));
+ container.getNodeId(), container.getContainerToken(), launcherId));
}
protected void unregisterAttemptFromListener(TezTaskAttemptID attemptId) {
- taskAttemptListener.unregisterTaskAttempt(attemptId);
+ taskAttemptListener.unregisterTaskAttempt(attemptId, taskCommId);
}
protected void registerAttemptWithListener(AMContainerTask amContainerTask) {
- taskAttemptListener.registerTaskAttempt(amContainerTask, this.containerId);
+ taskAttemptListener.registerTaskAttempt(amContainerTask, this.containerId, taskCommId);
}
protected void registerWithTAListener() {
- taskAttemptListener.registerRunningContainer(containerId);
+ taskAttemptListener.registerRunningContainer(containerId, taskCommId);
}
protected void unregisterFromTAListener() {
- this.taskAttemptListener.unregisterRunningContainer(containerId);
+ this.taskAttemptListener.unregisterRunningContainer(containerId, taskCommId);
}
protected void registerWithContainerListener() {
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
index 574c38e..938096d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
@@ -62,9 +62,9 @@ public class AMContainerMap extends AbstractService implements EventHandler<AMCo
}
}
- public boolean addContainerIfNew(Container container) {
+ public boolean addContainerIfNew(Container container, int schedulerId, int launcherId, int taskCommId) {
AMContainer amc = new AMContainerImpl(container, chh, tal,
- containerSignatureMatcher, context);
+ containerSignatureMatcher, context, schedulerId, launcherId, taskCommId);
return (containerMap.putIfAbsent(container.getId(), amc) == null);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
index b93cab3..0d8e4cd 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
@@ -257,7 +257,8 @@ public class AMNodeImpl implements AMNode {
// these containers are not useful anymore
pastContainers.addAll(containers);
containers.clear();
- sendEvent(new AMSchedulerEventNodeBlacklistUpdate(getNodeId(), true));
+ // TODO TEZ-2124 node tracking per ext source
+ sendEvent(new AMSchedulerEventNodeBlacklistUpdate(getNodeId(), true, 0));
}
@SuppressWarnings("unchecked")
@@ -363,7 +364,8 @@ public class AMNodeImpl implements AMNode {
public void transition(AMNodeImpl node, AMNodeEvent nEvent) {
node.ignoreBlacklisting = ignore;
if (node.getState() == AMNodeState.BLACKLISTED) {
- node.sendEvent(new AMSchedulerEventNodeBlacklistUpdate(node.getNodeId(), false));
+ // TODO TEZ-2124 node tracking per ext source
+ node.sendEvent(new AMSchedulerEventNodeBlacklistUpdate(node.getNodeId(), false, 0));
}
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index 26fc1ab..a466bc6 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -192,7 +192,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
@Override
public void serviceStart() throws Exception {
taListener = (TaskAttemptListenerImpTezDag) getTaskAttemptListener();
- taskCommunicator = (TezTaskCommunicatorImpl) taListener.getTaskCommunicator();
+ taskCommunicator = (TezTaskCommunicatorImpl) taListener.getTaskCommunicator(0);
eventHandlingThread = new Thread(this);
eventHandlingThread.start();
ExecutorService rawExecutor = Executors.newFixedThreadPool(handlerConcurrency,
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
index d98e5d1..318ee4e 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
@@ -74,7 +74,7 @@ public class TestTaskAttemptListenerImplTezDag {
TaskAttemptListenerImpTezDag taskAttemptListener =
new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
mock(ContainerHeartbeatHandler.class), null, null);
- TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator();
+ TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
@@ -93,37 +93,37 @@ public class TestTaskAttemptListenerImplTezDag {
ContainerId containerId2 = createContainerId(appId, 2);
ContainerContext containerContext2 = new ContainerContext(containerId2.toString());
- taskAttemptListener.registerRunningContainer(containerId2);
+ taskAttemptListener.registerRunningContainer(containerId2, 0);
containerTask = tezUmbilical.getTask(containerContext2);
assertNull(containerTask);
// Valid task registered
- taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2);
+ taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2, 0);
containerTask = tezUmbilical.getTask(containerContext2);
assertFalse(containerTask.shouldDie());
assertEquals(taskSpec, containerTask.getTaskSpec());
// Task unregistered. Should respond to heartbeats
- taskAttemptListener.unregisterTaskAttempt(taskAttemptId);
+ taskAttemptListener.unregisterTaskAttempt(taskAttemptId, 0);
containerTask = tezUmbilical.getTask(containerContext2);
assertNull(containerTask);
// Container unregistered. Should send a shouldDie = true
- taskAttemptListener.unregisterRunningContainer(containerId2);
+ taskAttemptListener.unregisterRunningContainer(containerId2, 0);
containerTask = tezUmbilical.getTask(containerContext2);
assertTrue(containerTask.shouldDie());
ContainerId containerId3 = createContainerId(appId, 3);
ContainerContext containerContext3 = new ContainerContext(containerId3.toString());
- taskAttemptListener.registerRunningContainer(containerId3);
+ taskAttemptListener.registerRunningContainer(containerId3, 0);
// Register task to container3, followed by unregistering container 3 all together
TaskSpec taskSpec2 = mock(TaskSpec.class);
TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec, null, null, false, 0);
- taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3);
- taskAttemptListener.unregisterRunningContainer(containerId3);
+ taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3, 0);
+ taskAttemptListener.unregisterRunningContainer(containerId3, 0);
containerTask = tezUmbilical.getTask(containerContext3);
assertTrue(containerTask.shouldDie());
}
@@ -154,7 +154,7 @@ public class TestTaskAttemptListenerImplTezDag {
TaskAttemptListenerImpTezDag taskAttemptListener =
new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
mock(ContainerHeartbeatHandler.class), null, null);
- TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator();
+ TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
@@ -167,12 +167,12 @@ public class TestTaskAttemptListenerImplTezDag {
ContainerId containerId1 = createContainerId(appId, 1);
ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
- taskAttemptListener.registerRunningContainer(containerId1);
+ taskAttemptListener.registerRunningContainer(containerId1, 0);
containerTask = tezUmbilical.getTask(containerContext1);
assertNull(containerTask);
// Register task
- taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1);
+ taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1, 0);
containerTask = tezUmbilical.getTask(containerContext1);
assertFalse(containerTask.shouldDie());
assertEquals(taskSpec, containerTask.getTaskSpec());
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index 2a2df7c..aa8cda8 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.tez.common.MockDNSToSwitchMapping;
+import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.api.TaskLocationHint;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
@@ -272,8 +273,9 @@ public class TestTaskAttempt {
MockEventHandler eventHandler = new MockEventHandler();
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(taListener).getTaskCommunicator(0);
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -322,8 +324,9 @@ public class TestTaskAttempt {
MockEventHandler eventHandler = spy(new MockEventHandler());
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(taListener).getTaskCommunicator(0);
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -344,7 +347,7 @@ public class TestTaskAttempt {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appCtx);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
doReturn(containers).when(appCtx).getAllContainers();
@@ -423,8 +426,9 @@ public class TestTaskAttempt {
MockEventHandler eventHandler = new MockEventHandler();
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(taListener).getTaskCommunicator(0);
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -445,7 +449,7 @@ public class TestTaskAttempt {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appCtx);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
doReturn(containers).when(appCtx).getAllContainers();
@@ -488,8 +492,9 @@ public class TestTaskAttempt {
MockEventHandler eventHandler = spy(new MockEventHandler());
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(taListener).getTaskCommunicator(0);
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -510,7 +515,7 @@ public class TestTaskAttempt {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appCtx);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
doReturn(containers).when(appCtx).getAllContainers();
@@ -580,8 +585,9 @@ public class TestTaskAttempt {
MockEventHandler eventHandler = spy(new MockEventHandler());
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(taListener).getTaskCommunicator(0);
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -603,7 +609,7 @@ public class TestTaskAttempt {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appCtx);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
doReturn(containers).when(appCtx).getAllContainers();
@@ -682,8 +688,9 @@ public class TestTaskAttempt {
MockEventHandler eventHandler = spy(new MockEventHandler());
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(taListener).getTaskCommunicator(0);
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -705,7 +712,7 @@ public class TestTaskAttempt {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appCtx);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
doReturn(containers).when(appCtx).getAllContainers();
@@ -774,8 +781,9 @@ public class TestTaskAttempt {
MockEventHandler eventHandler = spy(new MockEventHandler());
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(taListener).getTaskCommunicator(0);
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -796,7 +804,7 @@ public class TestTaskAttempt {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appCtx);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
doReturn(containers).when(appCtx).getAllContainers();
@@ -869,8 +877,9 @@ public class TestTaskAttempt {
MockEventHandler eventHandler = spy(new MockEventHandler());
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(taListener).getTaskCommunicator(0);
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -891,7 +900,7 @@ public class TestTaskAttempt {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appCtx);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
doReturn(containers).when(appCtx).getAllContainers();
@@ -972,8 +981,9 @@ public class TestTaskAttempt {
MockEventHandler eventHandler = spy(new MockEventHandler());
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(taListener).getTaskCommunicator(0);
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -994,7 +1004,7 @@ public class TestTaskAttempt {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appCtx);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
doReturn(containers).when(appCtx).getAllContainers();
@@ -1072,8 +1082,9 @@ public class TestTaskAttempt {
MockEventHandler mockEh = new MockEventHandler();
MockEventHandler eventHandler = spy(mockEh);
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(taListener).getTaskCommunicator(0);
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -1094,7 +1105,7 @@ public class TestTaskAttempt {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appCtx);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
doReturn(containers).when(appCtx).getAllContainers();
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 3147093..7275c7b 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -2170,7 +2170,7 @@ public class TestVertexImpl {
doReturn(dagId).when(appContext).getCurrentDAGID();
doReturn(dagId).when(dag).getID();
doReturn(taskScheduler).when(appContext).getTaskScheduler();
- doReturn(Resource.newInstance(102400, 60)).when(taskScheduler).getTotalResources();
+ doReturn(Resource.newInstance(102400, 60)).when(taskScheduler).getTotalResources(0);
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
@@ -2936,7 +2936,7 @@ public class TestVertexImpl {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appContext);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(containers).when(appContext).getAllContainers();
ta.handle(new TaskAttemptEventStartedRemotely(ta.getID(), contId, null));
@@ -2971,7 +2971,7 @@ public class TestVertexImpl {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appContext);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(containers).when(appContext).getAllContainers();
ta.handle(new TaskAttemptEventStartedRemotely(ta.getID(), contId, null));
@@ -3007,7 +3007,7 @@ public class TestVertexImpl {
AMContainerMap containers = new AMContainerMap(
mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
new ContainerContextMatcher(), appContext);
- containers.addContainerIfNew(container);
+ containers.addContainerIfNew(container, 0, 0, 0);
doReturn(containers).when(appContext).getAllContainers();
ta.handle(new TaskAttemptEventStartedRemotely(ta.getID(), contId, null));
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
index 54b9adb..c1169ef 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
@@ -223,7 +223,7 @@ public class TestContainerReuse {
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(
- ta11, containerHost1.getId(), TaskAttemptState.SUCCEEDED));
+ ta11, containerHost1.getId(), TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
verify(taskSchedulerEventHandler, times(1)).taskAllocated(
@@ -235,7 +235,7 @@ public class TestContainerReuse {
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta21, containerHost2.getId(),
- TaskAttemptState.SUCCEEDED));
+ TaskAttemptState.SUCCEEDED, 0));
long currentTs = System.currentTimeMillis();
Throwable exception = null;
@@ -356,7 +356,7 @@ public class TestContainerReuse {
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta21, containerHost2.getId(),
- TaskAttemptState.SUCCEEDED));
+ TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta21), eq(true));
verify(taskSchedulerEventHandler, times(0)).taskAllocated(
@@ -459,7 +459,7 @@ public class TestContainerReuse {
verify(taskSchedulerEventHandler).taskAllocated(eq(ta11), any(Object.class), eq(container1));
// Task assigned to container completed successfully. Container should be re-used.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta12), any(Object.class), eq(container1));
@@ -469,7 +469,7 @@ public class TestContainerReuse {
// Task assigned to container completed successfully.
// Verify reuse across hosts.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta12, container1.getId(), TaskAttemptState.SUCCEEDED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta12, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta12), eq(true));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta13), any(Object.class), eq(container1));
@@ -478,7 +478,7 @@ public class TestContainerReuse {
eventHandler.reset();
// Verify no re-use if a previous task fails.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta13, container1.getId(), TaskAttemptState.FAILED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta13, container1.getId(), TaskAttemptState.FAILED, 0));
drainableAppCallback.drain();
verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(ta14), any(Object.class), eq(container1));
verify(taskScheduler).deallocateTask(eq(ta13), eq(false));
@@ -496,7 +496,7 @@ public class TestContainerReuse {
verify(taskSchedulerEventHandler).taskAllocated(eq(ta14), any(Object.class), eq(container2));
// Task assigned to container completed successfully. No pending requests. Container should be released.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta14, container2.getId(), TaskAttemptState.SUCCEEDED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta14, container2.getId(), TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta14), eq(true));
verify(rmClient).releaseAssignedContainer(eq(container2.getId()));
@@ -607,7 +607,7 @@ public class TestContainerReuse {
// First task had profiling on. This container can not be reused further.
taskSchedulerEventHandler.handleEvent(
- new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED));
+ new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(ta12), any(Object.class),
@@ -653,7 +653,7 @@ public class TestContainerReuse {
// Verify that the container can not be reused when profiling option is turned on
// Even for 2 tasks having same profiling option can have container reusability.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta13, container2.getId(), TaskAttemptState.SUCCEEDED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta13, container2.getId(), TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta13), eq(true));
verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(ta14), any(Object.class),
@@ -698,7 +698,7 @@ public class TestContainerReuse {
verify(taskSchedulerEventHandler).taskAllocated(eq(ta15), any(Object.class), eq(container3));
//Ensure task 6 (of vertex 1) is allocated to same container
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta15, container3.getId(), TaskAttemptState.SUCCEEDED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta15, container3.getId(), TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta15), eq(true));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta16), any(Object.class), eq(container3));
@@ -811,7 +811,7 @@ public class TestContainerReuse {
// until delay expires.
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta11, container1.getId(),
- TaskAttemptState.SUCCEEDED));
+ TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
verify(taskSchedulerEventHandler, times(0)).taskAllocated(
@@ -828,7 +828,7 @@ public class TestContainerReuse {
// TA12 completed.
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta12, container1.getId(),
- TaskAttemptState.SUCCEEDED));
+ TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
LOG.info("Sleeping to ensure that the scheduling loop runs");
Thread.sleep(3000l);
@@ -946,7 +946,7 @@ public class TestContainerReuse {
// Container should be assigned to task21.
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta11, container1.getId(),
- TaskAttemptState.SUCCEEDED));
+ TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
verify(taskSchedulerEventHandler).taskAllocated(
@@ -956,7 +956,7 @@ public class TestContainerReuse {
// Task 2 completes.
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta21, container1.getId(),
- TaskAttemptState.SUCCEEDED));
+ TaskAttemptState.SUCCEEDED, 0));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
LOG.info("Sleeping to ensure that the scheduling loop runs");
@@ -1065,7 +1065,7 @@ public class TestContainerReuse {
assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
// Task assigned to container completed successfully. Container should be re-used.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta111), eq(true));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta112), any(Object.class), eq(container1));
@@ -1077,7 +1077,7 @@ public class TestContainerReuse {
// Task assigned to container completed successfully.
// Verify reuse across hosts.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta112), eq(true));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
@@ -1118,7 +1118,7 @@ public class TestContainerReuse {
assertEquals(2, assignEvent.getRemoteTaskLocalResources().size());
eventHandler.reset();
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta211, container1.getId(), TaskAttemptState.SUCCEEDED));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta211, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
drainableAppCallback.drain();
verify(taskScheduler).deallocateTask(eq(ta211), eq(true));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta212), any(Object.class), eq(container1));
[35/50] [abbrv] tez git commit: TEZ-2117. Add a manager for
ContainerLaunchers running in the AM. (sseth)
Posted by ss...@apache.org.
TEZ-2117. Add a manager for ContainerLaunchers running in the AM.
(sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/d2ceded9
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/d2ceded9
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/d2ceded9
Branch: refs/heads/TEZ-2003
Commit: d2ceded9b8517d4513f9edc90cbdbbffb146b975
Parents: 78d4ebc
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Feb 18 14:45:34 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:44:32 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 54 ++--------
.../tez/dag/app/launcher/ContainerLauncher.java | 2 +-
.../app/launcher/ContainerLauncherRouter.java | 108 +++++++++++++++++++
.../apache/tez/dag/app/MockDAGAppMaster.java | 5 +-
5 files changed, 124 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/d2ceded9/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 975ce65..1cd74a4 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -2,5 +2,6 @@ ALL CHANGES:
TEZ-2019. Temporarily allow the scheduler and launcher to be specified via configuration.
TEZ-2006. Task communication plane needs to be pluggable.
TEZ-2090. Add tests for jobs running in external services.
+ TEZ-2117. Add a manager for ContainerLaunchers running in the AM.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/d2ceded9/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index f9ec44d..4a3ce6d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -25,8 +25,6 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@@ -139,9 +137,7 @@ import org.apache.tez.dag.app.dag.event.TaskEventType;
import org.apache.tez.dag.app.dag.event.VertexEvent;
import org.apache.tez.dag.app.dag.event.VertexEventType;
import org.apache.tez.dag.app.dag.impl.DAGImpl;
-import org.apache.tez.dag.app.launcher.ContainerLauncher;
-import org.apache.tez.dag.app.launcher.ContainerLauncherImpl;
-import org.apache.tez.dag.app.launcher.LocalContainerLauncher;
+import org.apache.tez.dag.app.launcher.ContainerLauncherRouter;
import org.apache.tez.dag.app.rm.AMSchedulerEventType;
import org.apache.tez.dag.app.rm.NMCommunicatorEventType;
import org.apache.tez.dag.app.rm.TaskSchedulerEventHandler;
@@ -224,7 +220,7 @@ public class DAGAppMaster extends AbstractService {
private AppContext context;
private Configuration amConf;
private AsyncDispatcher dispatcher;
- private ContainerLauncher containerLauncher;
+ private ContainerLauncherRouter containerLauncherRouter;
private ContainerHeartbeatHandler containerHeartbeatHandler;
private TaskHeartbeatHandler taskHeartbeatHandler;
private TaskAttemptListener taskAttemptListener;
@@ -492,9 +488,9 @@ public class DAGAppMaster extends AbstractService {
taskSchedulerEventHandler);
addIfServiceDependency(taskSchedulerEventHandler, clientRpcServer);
- containerLauncher = createContainerLauncher(context);
- addIfService(containerLauncher, true);
- dispatcher.register(NMCommunicatorEventType.class, containerLauncher);
+ this.containerLauncherRouter = createContainerLauncherRouter(conf);
+ addIfService(containerLauncherRouter, true);
+ dispatcher.register(NMCommunicatorEventType.class, containerLauncherRouter);
historyEventHandler = createHistoryEventHandler(context);
addIfService(historyEventHandler, true);
@@ -1022,38 +1018,10 @@ public class DAGAppMaster extends AbstractService {
return chh;
}
- protected ContainerLauncher
- createContainerLauncher(final AppContext context) throws UnknownHostException {
- if(isLocal){
- LOG.info("Creating LocalContainerLauncher");
- return new LocalContainerLauncher(context, taskAttemptListener, workingDirectory);
- } else {
- // TODO: Temporary reflection with specific parameters until a clean interface is defined.
- String containerLauncherClassName = getConfig().get(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHER_CLASS);
- if (containerLauncherClassName == null) {
- LOG.info("Creating Default Container Launcher");
- return new ContainerLauncherImpl(context);
- } else {
- LOG.info("Creating container launcher : " + containerLauncherClassName);
- Class<? extends ContainerLauncher> containerLauncherClazz = (Class<? extends ContainerLauncher>) ReflectionUtils.getClazz(
- containerLauncherClassName);
- try {
- Constructor<? extends ContainerLauncher> ctor = containerLauncherClazz
- .getConstructor(AppContext.class, Configuration.class, TaskAttemptListener.class);
- ctor.setAccessible(true);
- ContainerLauncher instance = ctor.newInstance(context, getConfig(), taskAttemptListener);
- return instance;
- } catch (NoSuchMethodException e) {
- throw new TezUncheckedException(e);
- } catch (InvocationTargetException e) {
- throw new TezUncheckedException(e);
- } catch (InstantiationException e) {
- throw new TezUncheckedException(e);
- } catch (IllegalAccessException e) {
- throw new TezUncheckedException(e);
- }
- }
- }
+ protected ContainerLauncherRouter createContainerLauncherRouter(Configuration conf) throws
+ UnknownHostException {
+ return new ContainerLauncherRouter(conf, isLocal, context, taskAttemptListener, workingDirectory);
+
}
public ApplicationId getAppID() {
@@ -1076,8 +1044,8 @@ public class DAGAppMaster extends AbstractService {
return dispatcher;
}
- public ContainerLauncher getContainerLauncher() {
- return containerLauncher;
+ public ContainerLauncherRouter getContainerLauncherRouter() {
+ return containerLauncherRouter;
}
public TaskAttemptListener getTaskAttemptListener() {
http://git-wip-us.apache.org/repos/asf/tez/blob/d2ceded9/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncher.java
index 305eb50..8a8498f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncher.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
-public interface ContainerLauncher
+public interface ContainerLauncher
extends EventHandler<NMCommunicatorEvent> {
void dagComplete(DAG dag);
http://git-wip-us.apache.org/repos/asf/tez/blob/d2ceded9/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
new file mode 100644
index 0000000..34001ed
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.launcher;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.UnknownHostException;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
+
+public class ContainerLauncherRouter extends AbstractService
+ implements EventHandler<NMCommunicatorEvent> {
+
+ static final Log LOG = LogFactory.getLog(ContainerLauncherImpl.class);
+
+ private final ContainerLauncher containerLauncher;
+
+ @VisibleForTesting
+ public ContainerLauncherRouter(ContainerLauncher containerLauncher) {
+ super(ContainerLauncherRouter.class.getName());
+ this.containerLauncher = containerLauncher;
+ }
+
+ // Accepting conf to setup final parameters, if required.
+ public ContainerLauncherRouter(Configuration conf, boolean isLocal, AppContext context,
+ TaskAttemptListener taskAttemptListener,
+ String workingDirectory) throws UnknownHostException {
+ super(ContainerLauncherRouter.class.getName());
+
+ if (isLocal) {
+ LOG.info("Creating LocalContainerLauncher");
+ containerLauncher =
+ new LocalContainerLauncher(context, taskAttemptListener, workingDirectory);
+ } else {
+ // TODO: Temporary reflection with specific parameters until a clean interface is defined.
+ String containerLauncherClassName =
+ conf.get(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHER_CLASS);
+ if (containerLauncherClassName == null) {
+ LOG.info("Creating Default Container Launcher");
+ containerLauncher = new ContainerLauncherImpl(context);
+ } else {
+ LOG.info("Creating container launcher : " + containerLauncherClassName);
+ Class<? extends ContainerLauncher> containerLauncherClazz =
+ (Class<? extends ContainerLauncher>) ReflectionUtils.getClazz(
+ containerLauncherClassName);
+ try {
+ Constructor<? extends ContainerLauncher> ctor = containerLauncherClazz
+ .getConstructor(AppContext.class, Configuration.class, TaskAttemptListener.class);
+ ctor.setAccessible(true);
+ containerLauncher = ctor.newInstance(context, conf, taskAttemptListener);
+ } catch (NoSuchMethodException e) {
+ throw new TezUncheckedException(e);
+ } catch (InvocationTargetException e) {
+ throw new TezUncheckedException(e);
+ } catch (InstantiationException e) {
+ throw new TezUncheckedException(e);
+ } catch (IllegalAccessException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+
+ }
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ ((AbstractService)containerLauncher).init(conf);
+ }
+
+ @Override
+ public void serviceStart() {
+ ((AbstractService)containerLauncher).start();
+ }
+
+ @Override
+ public void serviceStop() {
+ ((AbstractService)containerLauncher).stop();
+ }
+
+
+ @Override
+ public void handle(NMCommunicatorEvent event) {
+ containerLauncher.handle(event);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d2ceded9/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index 0b2ea2f..2913cee 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -57,6 +57,7 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.launcher.ContainerLauncher;
+import org.apache.tez.dag.app.launcher.ContainerLauncherRouter;
import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
@@ -514,9 +515,9 @@ public class MockDAGAppMaster extends DAGAppMaster {
// use mock container launcher for tests
@Override
- protected ContainerLauncher createContainerLauncher(final AppContext context)
+ protected ContainerLauncherRouter createContainerLauncherRouter(final Configuration conf)
throws UnknownHostException {
- return containerLauncher;
+ return new ContainerLauncherRouter(containerLauncher);
}
@Override
[49/50] [abbrv] tez git commit: TEZ-2284. Separate TaskReporter into
an interface. (sseth)
Posted by ss...@apache.org.
TEZ-2284. Separate TaskReporter into an interface. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a9f4da1b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a9f4da1b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a9f4da1b
Branch: refs/heads/TEZ-2003
Commit: a9f4da1b62e7c192c359fe4ef29fa8e1bcf20f63
Parents: 623e8ac
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Apr 7 13:21:35 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:48:02 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../internals/api/TaskReporterInterface.java | 46 ++++++++++++++++++++
.../apache/tez/runtime/task/TaskReporter.java | 12 ++++-
.../org/apache/tez/runtime/task/TezChild.java | 3 +-
.../apache/tez/runtime/task/TezTaskRunner.java | 5 ++-
5 files changed, 62 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/a9f4da1b/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 6a4399c..e2c428d 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -13,5 +13,6 @@ ALL CHANGES:
TEZ-2187. Allow TaskCommunicators to report failed / killed attempts.
TEZ-2241. Miscellaneous fixes after last reabse.
TEZ-2283. Fixes after rebase 04/07.
+ TEZ-2284. Separate TaskReporter into an interface.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/a9f4da1b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TaskReporterInterface.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TaskReporterInterface.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TaskReporterInterface.java
new file mode 100644
index 0000000..47a61ab
--- /dev/null
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TaskReporterInterface.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.internals.api;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.RuntimeTask;
+import org.apache.tez.runtime.api.impl.EventMetaData;
+import org.apache.tez.runtime.api.impl.TezEvent;
+import org.apache.tez.runtime.task.ErrorReporter;
+
+public interface TaskReporterInterface {
+
+ // TODO TEZ-2003 Consolidate private API usage if making this public
+
+ void registerTask(RuntimeTask task, ErrorReporter errorReporter);
+
+ void unregisterTask(TezTaskAttemptID taskAttemptId);
+
+ boolean taskSucceeded(TezTaskAttemptID taskAttemptId) throws IOException, TezException;
+
+ boolean taskFailed(TezTaskAttemptID taskAttemptId, Throwable cause, String diagnostics, EventMetaData srcMeta) throws IOException,
+ TezException;
+
+ void addEvents(TezTaskAttemptID taskAttemptId, Collection<TezEvent> events);
+
+ boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException;
+
+ void shutdown();
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/a9f4da1b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java
index b9e7217..b1a3cf2 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java
@@ -45,6 +45,7 @@ import org.apache.tez.runtime.api.impl.TezEvent;
import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
import org.apache.tez.runtime.api.impl.EventMetaData.EventProducerConsumerType;
+import org.apache.tez.runtime.internals.api.TaskReporterInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,7 +64,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
* retrieve events specific to this task.
*
*/
-public class TaskReporter {
+public class TaskReporter implements TaskReporterInterface {
private static final Logger LOG = LoggerFactory.getLogger(TaskReporter.class);
@@ -95,6 +96,7 @@ public class TaskReporter {
/**
* Register a task to be tracked. Heartbeats will be sent out for this task to fetch events, etc.
*/
+ @Override
public synchronized void registerTask(RuntimeTask task,
ErrorReporter errorReporter) {
currentCallable = new HeartbeatCallable(task, umbilical, pollInterval, sendCounterInterval,
@@ -107,11 +109,13 @@ public class TaskReporter {
* This method should always be invoked before setting up heartbeats for another task running in
* the same container.
*/
+ @Override
public synchronized void unregisterTask(TezTaskAttemptID taskAttemptID) {
currentCallable.markComplete();
currentCallable = null;
}
-
+
+ @Override
public void shutdown() {
heartbeatExecutor.shutdownNow();
}
@@ -375,19 +379,23 @@ public class TaskReporter {
}
}
+ @Override
public synchronized boolean taskSucceeded(TezTaskAttemptID taskAttemptID) throws IOException, TezException {
return currentCallable.taskSucceeded(taskAttemptID);
}
+ @Override
public synchronized boolean taskFailed(TezTaskAttemptID taskAttemptID, Throwable t, String diagnostics,
EventMetaData srcMeta) throws IOException, TezException {
return currentCallable.taskFailed(taskAttemptID, t, diagnostics, srcMeta);
}
+ @Override
public synchronized void addEvents(TezTaskAttemptID taskAttemptID, Collection<TezEvent> events) {
currentCallable.addEvents(taskAttemptID, events);
}
+ @Override
public boolean canCommit(TezTaskAttemptID taskAttemptID) throws IOException {
return umbilical.canCommit(taskAttemptID);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/a9f4da1b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 7615f08..c4fd64c 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -66,6 +66,7 @@ import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
import org.apache.tez.runtime.api.impl.TaskSpec;
import org.apache.tez.runtime.api.impl.TezUmbilical;
import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl;
+import org.apache.tez.runtime.internals.api.TaskReporterInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -111,7 +112,7 @@ public class TezChild {
private final boolean ownUmbilical;
private final TezTaskUmbilicalProtocol umbilical;
- private TaskReporter taskReporter;
+ private TaskReporterInterface taskReporter;
private int taskCount = 0;
private TezVertexID lastVertexID;
http://git-wip-us.apache.org/repos/asf/tez/blob/a9f4da1b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
index 84ebffe..cf261d7 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
@@ -41,6 +41,7 @@ import org.apache.tez.runtime.api.impl.EventMetaData;
import org.apache.tez.runtime.api.impl.TaskSpec;
import org.apache.tez.runtime.api.impl.TezEvent;
import org.apache.tez.runtime.api.impl.TezUmbilical;
+import org.apache.tez.runtime.internals.api.TaskReporterInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,7 +57,7 @@ public class TezTaskRunner implements TezUmbilical, ErrorReporter {
private final LogicalIOProcessorRuntimeTask task;
private final UserGroupInformation ugi;
- private final TaskReporter taskReporter;
+ private final TaskReporterInterface taskReporter;
private final ListeningExecutorService executor;
private volatile ListenableFuture<Void> taskFuture;
private volatile Thread waitingThread;
@@ -70,7 +71,7 @@ public class TezTaskRunner implements TezUmbilical, ErrorReporter {
public TezTaskRunner(Configuration tezConf, UserGroupInformation ugi, String[] localDirs,
TaskSpec taskSpec, int appAttemptNumber,
Map<String, ByteBuffer> serviceConsumerMetadata, Map<String, String> serviceProviderEnvMap,
- Multimap<String, String> startedInputsMap, TaskReporter taskReporter,
+ Multimap<String, String> startedInputsMap, TaskReporterInterface taskReporter,
ListeningExecutorService executor, ObjectRegistry objectRegistry, String pid,
ExecutionContext executionContext, long memAvailable)
throws IOException {
[36/50] [abbrv] tez git commit: TEZ-2122. Setup pluggable components
at AM/Vertex level. (sseth)
Posted by ss...@apache.org.
TEZ-2122. Setup pluggable components at AM/Vertex level. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0ae9d4a3
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0ae9d4a3
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0ae9d4a3
Branch: refs/heads/TEZ-2003
Commit: 0ae9d4a3d5ec91607794c26d8a98e907352687fe
Parents: d2ceded
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Feb 19 14:59:18 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:45:10 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../apache/tez/dag/api/TezConfiguration.java | 29 +++-
.../org/apache/tez/dag/api/TezConstants.java | 3 +
.../java/org/apache/tez/dag/app/AppContext.java | 4 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 121 +++++++++++++-
.../dag/app/TaskAttemptListenerImpTezDag.java | 77 +++++----
.../java/org/apache/tez/dag/app/dag/Vertex.java | 4 +
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 8 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 47 ++++++
.../app/launcher/ContainerLauncherRouter.java | 93 +++++++----
.../app/rm/AMSchedulerEventTALaunchRequest.java | 22 ++-
.../dag/app/rm/TaskSchedulerEventHandler.java | 163 +++++++++++--------
.../apache/tez/dag/app/MockDAGAppMaster.java | 5 +-
.../app/TestTaskAttemptListenerImplTezDag.java | 13 +-
.../tez/dag/app/rm/TestContainerReuse.java | 2 +-
.../app/rm/TestTaskSchedulerEventHandler.java | 12 +-
.../dag/app/rm/TestTaskSchedulerHelpers.java | 18 +-
.../tez/tests/TestExternalTezServices.java | 19 ++-
18 files changed, 462 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 1cd74a4..4bfe08f 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -3,5 +3,6 @@ ALL CHANGES:
TEZ-2006. Task communication plane needs to be pluggable.
TEZ-2090. Add tests for jobs running in external services.
TEZ-2117. Add a manager for ContainerLaunchers running in the AM.
+ TEZ-2122. Setup pluggable components at AM/Vertex level.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index f71d9dd..fa361bf 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1136,13 +1136,36 @@ public class TezConfiguration extends Configuration {
+ "tez-ui.webservice.enable";
public static final boolean TEZ_AM_WEBSERVICE_ENABLE_DEFAULT = true;
+ /** defaults container-launcher for the specific vertex */
@ConfigurationScope(Scope.VERTEX)
- public static final String TEZ_AM_CONTAINER_LAUNCHER_CLASS = TEZ_AM_PREFIX + "container-launcher.class";
+ public static final String TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME = TEZ_AM_PREFIX + "vertex.container-launcher.name";
+ /** defaults task-scheduler for the specific vertex */
@ConfigurationScope(Scope.VERTEX)
- public static final String TEZ_AM_TASK_SCHEDULER_CLASS = TEZ_AM_PREFIX + "task-scheduler.class";
+ public static final String TEZ_AM_VERTEX_TASK_SCHEDULER_NAME = TEZ_AM_PREFIX + "vertex.task-scheduler.name";
+ /** defaults task-communicator for the specific vertex */
@ConfigurationScope(Scope.VERTEX)
- public static final String TEZ_AM_TASK_COMMUNICATOR_CLASS = TEZ_AM_PREFIX + "task-communicator.class";
+ public static final String TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME = TEZ_AM_PREFIX + "vertex.task-communicator.name";
+ /** Comma separated list of named container-launcher classes running in the AM.
+ * The format for each entry is NAME:CLASSNAME, except for tez default which is specified as Tez
+ * e.g. Tez, ExtService:org.apache.ExtLauncherClasss
+ * */
+ @ConfigurationScope(Scope.AM)
+ public static final String TEZ_AM_CONTAINER_LAUNCHERS = TEZ_AM_PREFIX + "container-launchers";
+
+ /** Comma separated list of task-schedulers classes running in the AM.
+ * The format for each entry is NAME:CLASSNAME, except for tez default which is specified as Tez
+ * e.g. Tez, ExtService:org.apache.ExtSchedulerClasss
+ */
+ @ConfigurationScope(Scope.AM)
+ public static final String TEZ_AM_TASK_SCHEDULERS = TEZ_AM_PREFIX + "task-schedulers";
+
+ /** Comma separated list of task-communicators classes running in the AM.
+ * The format for each entry is NAME:CLASSNAME, except for tez default which is specified as Tez
+ * e.g. Tez, ExtService:org.apache.ExtTaskCommClass
+ * */
+ @ConfigurationScope(Scope.AM)
+ public static final String TEZ_AM_TASK_COMMUNICATORS = TEZ_AM_PREFIX + "task-communicators";
// TODO only validate property here, value can also be validated if necessary
public static void validateProperty(String property, Scope usedScope) {
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
index bc4208f..3b07c59 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
@@ -102,4 +102,7 @@ public class TezConstants {
/// Version-related Environment variables
public static final String TEZ_CLIENT_VERSION_ENV = "TEZ_CLIENT_VERSION";
+
+ public static final String TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT = "Tez";
+ public static final String TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT = "TezLocal";
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index 37f7624..9463226 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -109,4 +109,8 @@ public interface AppContext {
String getAMUser();
Credentials getAppCredentials();
+
+ public Integer getTaskCommunicatorIdentifier(String name);
+ public Integer getTaskScheduerIdentifier(String name);
+ public Integer getContainerLauncherIdentifier(String name);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 4a3ce6d..d386e9e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -54,6 +54,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
@@ -264,7 +266,12 @@ public class DAGAppMaster extends AbstractService {
private ExecutorService rawExecutor;
private ListeningExecutorService execService;
-
+
+ // TODO May not need to be a bidi map
+ private final BiMap<String, Integer> taskSchedulers = HashBiMap.create();
+ private final BiMap<String, Integer> containerLaunchers = HashBiMap.create();
+ private final BiMap<String, Integer> taskCommunicators = HashBiMap.create();
+
/**
* set of already executed dag names.
*/
@@ -368,6 +375,29 @@ public class DAGAppMaster extends AbstractService {
this.isLocal = conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
+ String tezDefaultClassIdentifier =
+ isLocal ? TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT :
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT;
+
+ String[] taskSchedulerClassIdentifiers = parsePlugins(taskSchedulers,
+ conf.getTrimmedStrings(TezConfiguration.TEZ_AM_TASK_SCHEDULERS,
+ tezDefaultClassIdentifier),
+ TezConfiguration.TEZ_AM_TASK_SCHEDULERS);
+
+ String[] containerLauncherClassIdentifiers = parsePlugins(containerLaunchers,
+ conf.getTrimmedStrings(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHERS,
+ tezDefaultClassIdentifier),
+ TezConfiguration.TEZ_AM_CONTAINER_LAUNCHERS);
+
+ String[] taskCommunicatorClassIdentifiers = parsePlugins(taskCommunicators,
+ conf.getTrimmedStrings(TezConfiguration.TEZ_AM_TASK_COMMUNICATORS,
+ tezDefaultClassIdentifier),
+ TezConfiguration.TEZ_AM_TASK_COMMUNICATORS);
+
+ LOG.info(buildPluginComponentLog(taskSchedulerClassIdentifiers, taskSchedulers, "TaskSchedulers"));
+ LOG.info(buildPluginComponentLog(containerLauncherClassIdentifiers, containerLaunchers, "ContainerLaunchers"));
+ LOG.info(buildPluginComponentLog(taskCommunicatorClassIdentifiers, taskCommunicators, "TaskCommunicators"));
+
boolean disableVersionCheck = conf.getBoolean(
TezConfiguration.TEZ_AM_DISABLE_CLIENT_VERSION_CHECK,
TezConfiguration.TEZ_AM_DISABLE_CLIENT_VERSION_CHECK_DEFAULT);
@@ -433,7 +463,7 @@ public class DAGAppMaster extends AbstractService {
//service to handle requests to TaskUmbilicalProtocol
taskAttemptListener = createTaskAttemptListener(context,
- taskHeartbeatHandler, containerHeartbeatHandler);
+ taskHeartbeatHandler, containerHeartbeatHandler, taskCommunicatorClassIdentifiers);
addIfService(taskAttemptListener, true);
containerSignatureMatcher = createContainerSignatureMatcher();
@@ -470,7 +500,8 @@ public class DAGAppMaster extends AbstractService {
}
this.taskSchedulerEventHandler = new TaskSchedulerEventHandler(context,
- clientRpcServer, dispatcher.getEventHandler(), containerSignatureMatcher, webUIService);
+ clientRpcServer, dispatcher.getEventHandler(), containerSignatureMatcher, webUIService,
+ taskSchedulerClassIdentifiers);
addIfService(taskSchedulerEventHandler, true);
if (enableWebUIService()) {
@@ -488,7 +519,7 @@ public class DAGAppMaster extends AbstractService {
taskSchedulerEventHandler);
addIfServiceDependency(taskSchedulerEventHandler, clientRpcServer);
- this.containerLauncherRouter = createContainerLauncherRouter(conf);
+ this.containerLauncherRouter = createContainerLauncherRouter(conf, containerLauncherClassIdentifiers);
addIfService(containerLauncherRouter, true);
dispatcher.register(NMCommunicatorEventType.class, containerLauncherRouter);
@@ -995,9 +1026,9 @@ public class DAGAppMaster extends AbstractService {
}
protected TaskAttemptListener createTaskAttemptListener(AppContext context,
- TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh) {
+ TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh, String[] taskCommunicatorClasses) {
TaskAttemptListener lis =
- new TaskAttemptListenerImpTezDag(context, thh, chh,jobTokenSecretManager);
+ new TaskAttemptListenerImpTezDag(context, thh, chh,jobTokenSecretManager, taskCommunicatorClasses);
return lis;
}
@@ -1018,9 +1049,9 @@ public class DAGAppMaster extends AbstractService {
return chh;
}
- protected ContainerLauncherRouter createContainerLauncherRouter(Configuration conf) throws
+ protected ContainerLauncherRouter createContainerLauncherRouter(Configuration conf, String []containerLauncherClasses) throws
UnknownHostException {
- return new ContainerLauncherRouter(conf, isLocal, context, taskAttemptListener, workingDirectory);
+ return new ContainerLauncherRouter(conf, context, taskAttemptListener, workingDirectory, containerLauncherClasses);
}
@@ -1447,6 +1478,21 @@ public class DAGAppMaster extends AbstractService {
}
@Override
+ public Integer getTaskCommunicatorIdentifier(String name) {
+ return taskCommunicators.get(name);
+ }
+
+ @Override
+ public Integer getTaskScheduerIdentifier(String name) {
+ return taskSchedulers.get(name);
+ }
+
+ @Override
+ public Integer getContainerLauncherIdentifier(String name) {
+ return taskCommunicators.get(name);
+ }
+
+ @Override
public Map<ApplicationAccessType, String> getApplicationACLs() {
if (getServiceState() != STATE.STARTED) {
throw new TezUncheckedException(
@@ -2196,4 +2242,63 @@ public class DAGAppMaster extends AbstractService {
return amConf.getBoolean(TezConfiguration.TEZ_AM_WEBSERVICE_ENABLE,
TezConfiguration.TEZ_AM_WEBSERVICE_ENABLE_DEFAULT);
}
+
+ // Tez default classnames are populated as TezConfiguration.TEZ_AM_SERVICE_PLUGINS_DEFAULT
+ private String[] parsePlugins(BiMap<String, Integer> pluginMap, String[] pluginStrings,
+ String context) {
+ Preconditions.checkState(pluginStrings != null && pluginStrings.length > 0,
+ "Plugin strings should not be null or empty: " + context);
+
+ String[] classNames = new String[pluginStrings.length];
+
+ int index = 0;
+ for (String pluginString : pluginStrings) {
+
+ String className;
+ String identifierString;
+
+ Preconditions.checkState(pluginString != null && !pluginString.isEmpty(),
+ "Plugin string: " + pluginString + " should not be null or empty");
+ if (pluginString.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT) ||
+ pluginString.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT)) {
+ // Kind of ugly, but Tez internal routing is encoded via a String instead of classnames.
+ // Individual components - TaskComm, Scheduler, Launcher deal with actual classname translation,
+ // and avoid reflection.
+ identifierString = pluginString;
+ className = pluginString;
+ } else {
+ String[] parts = pluginString.split(":");
+ Preconditions.checkState(
+ parts.length == 2 && parts[0] != null && !parts[0].isEmpty() && parts[1] != null &&
+ !parts[1].isEmpty(),
+ "Invalid configuration string for " + context + ": " + pluginString);
+ Preconditions.checkState(
+ !parts[0].equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT) &&
+ !parts[0].equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT),
+ "Identifier cannot be " + TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT + " or " +
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT + " for " +
+ pluginString);
+ identifierString = parts[0];
+ className = parts[1];
+ }
+ pluginMap.put(identifierString, index);
+ classNames[index] = className;
+ }
+ return classNames;
+ }
+
+ String buildPluginComponentLog(String[] classIdentifiers, BiMap<String, Integer> map,
+ String component) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("AM Level configured ").append(component).append(": ");
+ for (int i = 0; i < classIdentifiers.length; i++) {
+ sb.append("[").append(i).append(":").append(map.inverse().get(i)).append(":")
+ .append(taskSchedulers.inverse().get(i)).append(
+ "]");
+ if (i != classIdentifiers.length - 1) {
+ sb.append(",");
+ }
+ }
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 68b5a75..94e4f71 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -28,7 +28,6 @@ import java.util.concurrent.ConcurrentMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -37,7 +36,7 @@ import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.api.TaskCommunicatorContext;
import org.apache.tez.dag.api.TaskHeartbeatResponse;
-import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -47,7 +46,6 @@ import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
-import org.apache.tez.dag.app.rm.TaskSchedulerService;
import org.apache.tez.dag.app.rm.container.AMContainerTask;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezVertexID;
@@ -64,7 +62,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
.getLogger(TaskAttemptListenerImpTezDag.class);
private final AppContext context;
- private TaskCommunicator taskCommunicator;
+ private final TaskCommunicator[] taskCommunicators;
protected final TaskHeartbeatHandler taskHeartbeatHandler;
protected final ContainerHeartbeatHandler containerHeartbeatHandler;
@@ -90,28 +88,52 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
public TaskAttemptListenerImpTezDag(AppContext context,
TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
// TODO TEZ-2003 pre-merge. Remove reference to JobTokenSecretManager.
- JobTokenSecretManager jobTokenSecretManager) {
+ JobTokenSecretManager jobTokenSecretManager,
+ String [] taskCommunicatorClassIdentifiers) {
super(TaskAttemptListenerImpTezDag.class.getName());
this.context = context;
this.taskHeartbeatHandler = thh;
this.containerHeartbeatHandler = chh;
- this.taskCommunicator = new TezTaskCommunicatorImpl(this);
+ if (taskCommunicatorClassIdentifiers == null || taskCommunicatorClassIdentifiers.length == 0) {
+ taskCommunicatorClassIdentifiers = new String[] {TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT};
+ }
+ this.taskCommunicators = new TaskCommunicator[taskCommunicatorClassIdentifiers.length];
+ for (int i = 0 ; i < taskCommunicatorClassIdentifiers.length ; i++) {
+ taskCommunicators[i] = createTaskCommunicator(taskCommunicatorClassIdentifiers[i]);
+ }
+ // TODO TEZ-2118 Start using taskCommunicator indices properly
+ }
+
+ @Override
+ public void serviceStart() {
+ // TODO Why is init tied to serviceStart
+ for (int i = 0 ; i < taskCommunicators.length ; i++) {
+ taskCommunicators[i].init(getConfig());
+ taskCommunicators[i].start();
+ }
}
@Override
- public void serviceInit(Configuration conf) {
- String taskCommClassName = conf.get(TezConfiguration.TEZ_AM_TASK_COMMUNICATOR_CLASS);
- if (taskCommClassName == null) {
+ public void serviceStop() {
+ for (int i = 0 ; i < taskCommunicators.length ; i++) {
+ taskCommunicators[i].stop();
+ }
+ }
+
+ private TaskCommunicator createTaskCommunicator(String taskCommClassIdentifier) {
+ if (taskCommClassIdentifier.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT) ||
+ taskCommClassIdentifier
+ .equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT)) {
LOG.info("Using Default Task Communicator");
- this.taskCommunicator = new TezTaskCommunicatorImpl(this);
+ return new TezTaskCommunicatorImpl(this);
} else {
- LOG.info("Using TaskCommunicator: " + taskCommClassName);
+ LOG.info("Using TaskCommunicator: " + taskCommClassIdentifier);
Class<? extends TaskCommunicator> taskCommClazz = (Class<? extends TaskCommunicator>) ReflectionUtils
- .getClazz(taskCommClassName);
+ .getClazz(taskCommClassIdentifier);
try {
Constructor<? extends TaskCommunicator> ctor = taskCommClazz.getConstructor(TaskCommunicatorContext.class);
ctor.setAccessible(true);
- this.taskCommunicator = ctor.newInstance(this);
+ return ctor.newInstance(this);
} catch (NoSuchMethodException e) {
throw new TezUncheckedException(e);
} catch (InvocationTargetException e) {
@@ -125,20 +147,6 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
@Override
- public void serviceStart() {
- taskCommunicator.init(getConfig());
- taskCommunicator.start();
- }
-
- @Override
- public void serviceStop() {
- if (taskCommunicator != null) {
- taskCommunicator.stop();
- taskCommunicator = null;
- }
- }
-
- @Override
public ApplicationAttemptId getApplicationAttemptId() {
return context.getApplicationAttemptId();
}
@@ -214,7 +222,8 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
@Override
public void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId) {
- context.getEventHandler().handle(new TaskAttemptEventStartedRemotely(taskAttemptID, containerId, null));
+ context.getEventHandler()
+ .handle(new TaskAttemptEventStartedRemotely(taskAttemptID, containerId, null));
pingContainerHeartbeatHandler(containerId);
}
@@ -244,7 +253,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
@Override
public InetSocketAddress getAddress() {
- return taskCommunicator.getAddress();
+ return taskCommunicators[0].getAddress();
}
// The TaskAttemptListener register / unregister methods in this class are not thread safe.
@@ -276,7 +285,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
"Multiple registrations for containerId: " + containerId);
}
NodeId nodeId = context.getAllContainers().get(containerId).getContainer().getNodeId();
- taskCommunicator.registerRunningContainer(containerId, nodeId.getHost(), nodeId.getPort());
+ taskCommunicators[0].registerRunningContainer(containerId, nodeId.getHost(), nodeId.getPort());
}
@Override
@@ -288,7 +297,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
if (containerInfo.taskAttemptId != null) {
registeredAttempts.remove(containerInfo.taskAttemptId);
}
- taskCommunicator.registerContainerEnd(containerId);
+ taskCommunicators[0].registerContainerEnd(containerId);
}
@Override
@@ -323,7 +332,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
+ amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+ " when already assigned to: " + containerIdFromMap);
}
- taskCommunicator.registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
+ taskCommunicators[0].registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
amContainerTask.getAdditionalResources(), amContainerTask.getCredentials(),
amContainerTask.haveCredentialsChanged());
}
@@ -343,7 +352,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
// Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
registeredContainers.put(containerId, NULL_CONTAINER_INFO);
- taskCommunicator.unregisterRunningTaskAttempt(attemptId);
+ taskCommunicators[0].unregisterRunningTaskAttempt(attemptId);
}
private void pingContainerHeartbeatHandler(ContainerId containerId) {
@@ -361,6 +370,6 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
public TaskCommunicator getTaskCommunicator() {
- return taskCommunicator;
+ return taskCommunicators[0];
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index 77ef6e0..3fa3669 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -168,4 +168,8 @@ public interface Vertex extends Comparable<Vertex> {
public int getKilledTaskAttemptCount();
public Configuration getConf();
+
+ public int getTaskSchedulerIdentifier();
+ public int getContainerLauncherIdentifier();
+ public int getTaskCommunicatorIdentifier();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 1af4274..b5a6c9c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -1062,9 +1062,15 @@ public class TaskAttemptImpl implements TaskAttempt,
priority = (scheduleEvent.getPriorityHighLimit() + scheduleEvent.getPriorityLowLimit()) / 2;
}
+ // TODO Jira post TEZ-2003 getVertex implementation is very inefficient. This should be via references, instead of locked table lookups.
+ Vertex vertex = ta.getVertex();
AMSchedulerEventTALaunchRequest launchRequestEvent = new AMSchedulerEventTALaunchRequest(
ta.attemptId, ta.taskResource, remoteTaskSpec, ta, locationHint,
- priority, ta.containerContext);
+ priority, ta.containerContext,
+ vertex.getTaskSchedulerIdentifier(),
+ vertex.getContainerLauncherIdentifier(),
+ vertex.getTaskCommunicatorIdentifier());
+
ta.sendEvent(launchRequestEvent);
return TaskAttemptStateInternal.START_WAIT;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index e22343b..bce4127 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -73,6 +73,7 @@ import org.apache.tez.dag.api.ProcessorDescriptor;
import org.apache.tez.dag.api.RootInputLeafOutput;
import org.apache.tez.dag.api.Scope;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.TaskLocationHint;
@@ -228,6 +229,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private final boolean isSpeculationEnabled;
+ private final int taskSchedulerIdentifier;
+ private final int containerLauncherIdentifier;
+ private final int taskCommunicatorIdentifier;
+
//fields initialized in init
@VisibleForTesting
@@ -936,6 +941,33 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// This "this leak" is okay because the retained pointer is in an
// instance variable.
+ boolean isLocal = vertexConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
+ TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
+
+ String tezDefaultComponentName =
+ isLocal ? TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT :
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT;
+ String taskSchedulerName =
+ vertexConf.get(TezConfiguration.TEZ_AM_VERTEX_TASK_SCHEDULER_NAME, tezDefaultComponentName);
+ String taskCommName = vertexConf
+ .get(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME, tezDefaultComponentName);
+ String containerLauncherName = vertexConf
+ .get(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME, tezDefaultComponentName);
+ taskSchedulerIdentifier = appContext.getTaskScheduerIdentifier(taskSchedulerName);
+ taskCommunicatorIdentifier = appContext.getTaskCommunicatorIdentifier(taskCommName);
+ containerLauncherIdentifier = appContext.getContainerLauncherIdentifier(containerLauncherName);
+
+ Preconditions.checkNotNull(taskSchedulerIdentifier, "Unknown taskScheduler: " + taskSchedulerName);
+ Preconditions.checkNotNull(taskCommunicatorIdentifier, "Unknown taskCommunicator: " + containerLauncherName);
+ Preconditions.checkNotNull(containerLauncherIdentifier, "Unknown containerLauncher: " + taskCommName);
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("Running vertex: ").append(logIdentifier).append(" : ")
+ .append("TaskScheduler=").append(taskSchedulerIdentifier).append(":").append(taskSchedulerName)
+ .append(", ContainerLauncher=").append(containerLauncherIdentifier).append(":").append(containerLauncherName)
+ .append(", TaskCommunicator=").append(taskCommunicatorIdentifier).append(":").append(taskCommName);
+ LOG.info(sb.toString());
+
stateMachine = new StateMachineTez<VertexState, VertexEventType, VertexEvent, VertexImpl>(
stateMachineFactory.make(this), this);
augmentStateMachine();
@@ -946,6 +978,21 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
return vertexConf;
}
+ @Override
+ public int getTaskSchedulerIdentifier() {
+ return this.taskSchedulerIdentifier;
+ }
+
+ @Override
+ public int getContainerLauncherIdentifier() {
+ return this.containerLauncherIdentifier;
+ }
+
+ @Override
+ public int getTaskCommunicatorIdentifier() {
+ return this.taskCommunicatorIdentifier;
+ }
+
private boolean isSpeculationEnabled() {
return isSpeculationEnabled;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
index 34001ed..621e4a8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
@@ -36,73 +37,93 @@ public class ContainerLauncherRouter extends AbstractService
static final Log LOG = LogFactory.getLog(ContainerLauncherImpl.class);
- private final ContainerLauncher containerLauncher;
+ private final ContainerLauncher containerLaunchers[];
@VisibleForTesting
public ContainerLauncherRouter(ContainerLauncher containerLauncher) {
super(ContainerLauncherRouter.class.getName());
- this.containerLauncher = containerLauncher;
+ containerLaunchers = new ContainerLauncher[] {containerLauncher};
}
// Accepting conf to setup final parameters, if required.
- public ContainerLauncherRouter(Configuration conf, boolean isLocal, AppContext context,
+ public ContainerLauncherRouter(Configuration conf, AppContext context,
TaskAttemptListener taskAttemptListener,
- String workingDirectory) throws UnknownHostException {
+ String workingDirectory,
+ String[] containerLauncherClassIdentifiers) throws UnknownHostException {
super(ContainerLauncherRouter.class.getName());
- if (isLocal) {
+ if (containerLauncherClassIdentifiers == null || containerLauncherClassIdentifiers.length == 0) {
+ containerLauncherClassIdentifiers = new String[] {TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT};
+ }
+ containerLaunchers = new ContainerLauncher[containerLauncherClassIdentifiers.length];
+
+ for (int i = 0; i < containerLauncherClassIdentifiers.length; i++) {
+ containerLaunchers[i] = createContainerLauncher(containerLauncherClassIdentifiers[i], context,
+ taskAttemptListener, workingDirectory, conf);
+ }
+ }
+
+ private ContainerLauncher createContainerLauncher(String containerLauncherClassIdentifier,
+ AppContext context,
+ TaskAttemptListener taskAttemptListener,
+ String workingDirectory,
+ Configuration conf) throws
+ UnknownHostException {
+ if (containerLauncherClassIdentifier.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT)) {
+ LOG.info("Creating DefaultContainerLauncher");
+ return new ContainerLauncherImpl(context);
+ } else if (containerLauncherClassIdentifier
+ .equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT)) {
LOG.info("Creating LocalContainerLauncher");
- containerLauncher =
+ return
new LocalContainerLauncher(context, taskAttemptListener, workingDirectory);
} else {
- // TODO: Temporary reflection with specific parameters until a clean interface is defined.
- String containerLauncherClassName =
- conf.get(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHER_CLASS);
- if (containerLauncherClassName == null) {
- LOG.info("Creating Default Container Launcher");
- containerLauncher = new ContainerLauncherImpl(context);
- } else {
- LOG.info("Creating container launcher : " + containerLauncherClassName);
- Class<? extends ContainerLauncher> containerLauncherClazz =
- (Class<? extends ContainerLauncher>) ReflectionUtils.getClazz(
- containerLauncherClassName);
- try {
- Constructor<? extends ContainerLauncher> ctor = containerLauncherClazz
- .getConstructor(AppContext.class, Configuration.class, TaskAttemptListener.class);
- ctor.setAccessible(true);
- containerLauncher = ctor.newInstance(context, conf, taskAttemptListener);
- } catch (NoSuchMethodException e) {
- throw new TezUncheckedException(e);
- } catch (InvocationTargetException e) {
- throw new TezUncheckedException(e);
- } catch (InstantiationException e) {
- throw new TezUncheckedException(e);
- } catch (IllegalAccessException e) {
- throw new TezUncheckedException(e);
- }
+ LOG.info("Creating container launcher : " + containerLauncherClassIdentifier);
+ Class<? extends ContainerLauncher> containerLauncherClazz =
+ (Class<? extends ContainerLauncher>) ReflectionUtils.getClazz(
+ containerLauncherClassIdentifier);
+ try {
+ Constructor<? extends ContainerLauncher> ctor = containerLauncherClazz
+ .getConstructor(AppContext.class, Configuration.class, TaskAttemptListener.class);
+ ctor.setAccessible(true);
+ return ctor.newInstance(context, conf, taskAttemptListener);
+ } catch (NoSuchMethodException e) {
+ throw new TezUncheckedException(e);
+ } catch (InvocationTargetException e) {
+ throw new TezUncheckedException(e);
+ } catch (InstantiationException e) {
+ throw new TezUncheckedException(e);
+ } catch (IllegalAccessException e) {
+ throw new TezUncheckedException(e);
}
-
}
+ // TODO TEZ-2118 Handle routing to multiple launchers
}
@Override
public void serviceInit(Configuration conf) {
- ((AbstractService)containerLauncher).init(conf);
+ for (int i = 0 ; i < containerLaunchers.length ; i++) {
+ ((AbstractService) containerLaunchers[i]).init(conf);
+ }
}
@Override
public void serviceStart() {
- ((AbstractService)containerLauncher).start();
+ for (int i = 0 ; i < containerLaunchers.length ; i++) {
+ ((AbstractService) containerLaunchers[i]).start();
+ }
}
@Override
public void serviceStop() {
- ((AbstractService)containerLauncher).stop();
+ for (int i = 0 ; i < containerLaunchers.length ; i++) {
+ ((AbstractService) containerLaunchers[i]).stop();
+ }
}
@Override
public void handle(NMCommunicatorEvent event) {
- containerLauncher.handle(event);
+ containerLaunchers[0].handle(event);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
index 5c4d43c..c59193c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
@@ -38,11 +38,16 @@ public class AMSchedulerEventTALaunchRequest extends AMSchedulerEvent {
private final TaskSpec remoteTaskSpec;
private final TaskAttempt taskAttempt;
+ private final int schedulerId;
+ private final int launcherId;
+ private final int taskCommId;
+
public AMSchedulerEventTALaunchRequest(TezTaskAttemptID attemptId,
Resource capability,
TaskSpec remoteTaskSpec, TaskAttempt ta,
TaskLocationHint locationHint, int priority,
- ContainerContext containerContext) {
+ ContainerContext containerContext,
+ int schedulerId, int launcherId, int taskCommId) {
super(AMSchedulerEventType.S_TA_LAUNCH_REQUEST);
this.attemptId = attemptId;
this.capability = capability;
@@ -51,6 +56,9 @@ public class AMSchedulerEventTALaunchRequest extends AMSchedulerEvent {
this.locationHint = locationHint;
this.priority = priority;
this.containerContext = containerContext;
+ this.schedulerId = schedulerId;
+ this.launcherId = launcherId;
+ this.taskCommId = taskCommId;
}
public TezTaskAttemptID getAttemptID() {
@@ -81,6 +89,18 @@ public class AMSchedulerEventTALaunchRequest extends AMSchedulerEvent {
return this.containerContext;
}
+ public int getSchedulerId() {
+ return schedulerId;
+ }
+
+ public int getLauncherId() {
+ return launcherId;
+ }
+
+ public int getTaskCommId() {
+ return taskCommId;
+ }
+
// Parameter replacement: @taskid@ will not be usable
// ProfileTaskRange not available along with ContainerReUse
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 8c3ed87..72389e7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.TaskLocationHint;
import org.apache.tez.dag.api.TaskLocationHint.TaskBasedLocationAffinity;
@@ -92,7 +93,6 @@ public class TaskSchedulerEventHandler extends AbstractService
@SuppressWarnings("rawtypes")
private final EventHandler eventHandler;
private final String historyUrl;
- protected TaskSchedulerService taskScheduler;
private DAGAppMaster dagAppMaster;
private Map<ApplicationAccessType, String> appAcls = null;
private Thread eventHandlingThread;
@@ -105,14 +105,27 @@ public class TaskSchedulerEventHandler extends AbstractService
private AtomicBoolean shouldUnregisterFlag =
new AtomicBoolean(false);
private final WebUIService webUI;
+ private final String[] taskSchedulerClasses;
+ protected final TaskSchedulerService []taskSchedulers;
BlockingQueue<AMSchedulerEvent> eventQueue
= new LinkedBlockingQueue<AMSchedulerEvent>();
+ /**
+ *
+ * @param appContext
+ * @param clientService
+ * @param eventHandler
+ * @param containerSignatureMatcher
+ * @param webUI
+ * @param schedulerClasses the list of scheduler classes / codes. Tez internal classes are represented as codes.
+ * An empty list defaults to using the YarnTaskScheduler as the only source.
+ */
@SuppressWarnings("rawtypes")
public TaskSchedulerEventHandler(AppContext appContext,
DAGClientServer clientService, EventHandler eventHandler,
- ContainerSignatureMatcher containerSignatureMatcher, WebUIService webUI) {
+ ContainerSignatureMatcher containerSignatureMatcher, WebUIService webUI,
+ String [] schedulerClasses) {
super(TaskSchedulerEventHandler.class.getName());
this.appContext = appContext;
this.eventHandler = eventHandler;
@@ -123,6 +136,12 @@ public class TaskSchedulerEventHandler extends AbstractService
if (this.webUI != null) {
this.webUI.setHistoryUrl(this.historyUrl);
}
+ if (schedulerClasses == null || schedulerClasses.length == 0) {
+ this.taskSchedulerClasses = new String[] {TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT};
+ } else {
+ this.taskSchedulerClasses = schedulerClasses;
+ }
+ taskSchedulers = new TaskSchedulerService[this.taskSchedulerClasses.length];
}
public Map<ApplicationAccessType, String> getApplicationAcls() {
@@ -139,11 +158,11 @@ public class TaskSchedulerEventHandler extends AbstractService
}
public Resource getAvailableResources() {
- return taskScheduler.getAvailableResources();
+ return taskSchedulers[0].getAvailableResources();
}
public Resource getTotalResources() {
- return taskScheduler.getTotalResources();
+ return taskSchedulers[0].getTotalResources();
}
public synchronized void handleEvent(AMSchedulerEvent sEvent) {
@@ -209,9 +228,9 @@ public class TaskSchedulerEventHandler extends AbstractService
private void handleNodeBlacklistUpdate(AMSchedulerEventNodeBlacklistUpdate event) {
if (event.getType() == AMSchedulerEventType.S_NODE_BLACKLISTED) {
- taskScheduler.blacklistNode(event.getNodeId());
+ taskSchedulers[0].blacklistNode(event.getNodeId());
} else if (event.getType() == AMSchedulerEventType.S_NODE_UNBLACKLISTED) {
- taskScheduler.unblacklistNode(event.getNodeId());
+ taskSchedulers[0].unblacklistNode(event.getNodeId());
} else {
throw new TezUncheckedException("Invalid event type: " + event.getType());
}
@@ -223,14 +242,14 @@ public class TaskSchedulerEventHandler extends AbstractService
// TODO what happens to the task that was connected to this container?
// current assumption is that it will eventually call handleTaStopRequest
//TaskAttempt taskAttempt = (TaskAttempt)
- taskScheduler.deallocateContainer(containerId);
+ taskSchedulers[0].deallocateContainer(containerId);
// TODO does this container need to be stopped via C_STOP_REQUEST
sendEvent(new AMContainerEventStopRequest(containerId));
}
private void handleTAUnsuccessfulEnd(AMSchedulerEventTAEnded event) {
TaskAttempt attempt = event.getAttempt();
- boolean wasContainerAllocated = taskScheduler.deallocateTask(attempt, false);
+ boolean wasContainerAllocated = taskSchedulers[0].deallocateTask(attempt, false);
// use stored value of container id in case the scheduler has removed this
// assignment because the task has been deallocated earlier.
// retroactive case
@@ -272,7 +291,7 @@ public class TaskSchedulerEventHandler extends AbstractService
event.getAttemptID()));
}
- boolean wasContainerAllocated = taskScheduler.deallocateTask(attempt, true);
+ boolean wasContainerAllocated = taskSchedulers[0].deallocateTask(attempt, true);
if (!wasContainerAllocated) {
LOG.error("De-allocated successful task: " + attempt.getID()
+ ", but TaskScheduler reported no container assigned to task");
@@ -297,7 +316,7 @@ public class TaskSchedulerEventHandler extends AbstractService
TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt();
if (affinityAttempt != null) {
Preconditions.checkNotNull(affinityAttempt.getAssignedContainerID(), affinityAttempt.getID());
- taskScheduler.allocateTask(taskAttempt,
+ taskSchedulers[0].allocateTask(taskAttempt,
event.getCapability(),
affinityAttempt.getAssignedContainerID(),
Priority.newInstance(event.getPriority()),
@@ -316,57 +335,59 @@ public class TaskSchedulerEventHandler extends AbstractService
.toArray(new String[locationHint.getRacks().size()]) : null;
}
}
-
- taskScheduler.allocateTask(taskAttempt,
- event.getCapability(),
- hosts,
- racks,
- Priority.newInstance(event.getPriority()),
- event.getContainerContext(),
- event);
- }
-
-
- protected TaskSchedulerService createTaskScheduler(String host, int port,
- String trackingUrl, AppContext appContext) {
- boolean isLocal = getConfig().getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
- TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
- if (isLocal) {
- LOG.info("Using TaskScheduler: LocalTaskSchedulerService");
+
+ taskSchedulers[0].allocateTask(taskAttempt,
+ event.getCapability(),
+ hosts,
+ racks,
+ Priority.newInstance(event.getPriority()),
+ event.getContainerContext(),
+ event);
+ }
+
+ private TaskSchedulerService createTaskScheduler(String host, int port, String trackingUrl,
+ AppContext appContext,
+ String schedulerClassName) {
+ if (schedulerClassName.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT)) {
+ LOG.info("Creating TaskScheduler: YarnTaskSchedulerService");
+ return new YarnTaskSchedulerService(this, this.containerSignatureMatcher,
+ host, port, trackingUrl, appContext);
+ } else if (schedulerClassName.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT)) {
+ LOG.info("Creating TaskScheduler: Local TaskScheduler");
return new LocalTaskSchedulerService(this, this.containerSignatureMatcher,
host, port, trackingUrl, appContext);
- }
- else {
- String schedulerClassName = getConfig().get(TezConfiguration.TEZ_AM_TASK_SCHEDULER_CLASS);
- if (schedulerClassName == null) {
- LOG.info("Using TaskScheduler: YarnTaskSchedulerService");
- return new YarnTaskSchedulerService(this, this.containerSignatureMatcher,
- host, port, trackingUrl, appContext);
- } else {
- LOG.info("Using custom TaskScheduler: " + schedulerClassName);
- // TODO Temporary reflection with specific parameters. Remove once there is a clean interface.
- Class<? extends TaskSchedulerService> taskSchedulerClazz =
- (Class<? extends TaskSchedulerService>) ReflectionUtils.getClazz(schedulerClassName);
- try {
- Constructor<? extends TaskSchedulerService> ctor = taskSchedulerClazz
- .getConstructor(TaskSchedulerAppCallback.class, AppContext.class, String.class,
- int.class, String.class, Configuration.class);
- ctor.setAccessible(true);
- TaskSchedulerService taskSchedulerService =
- ctor.newInstance(this, appContext, host, port, trackingUrl, getConfig());
- return taskSchedulerService;
- } catch (NoSuchMethodException e) {
- throw new TezUncheckedException(e);
- } catch (InvocationTargetException e) {
- throw new TezUncheckedException(e);
- } catch (InstantiationException e) {
- throw new TezUncheckedException(e);
- } catch (IllegalAccessException e) {
- throw new TezUncheckedException(e);
- }
+ } else {
+ LOG.info("Creating custom TaskScheduler: " + schedulerClassName);
+ // TODO TEZ-2003 Temporary reflection with specific parameters. Remove once there is a clean interface.
+ Class<? extends TaskSchedulerService> taskSchedulerClazz =
+ (Class<? extends TaskSchedulerService>) ReflectionUtils.getClazz(schedulerClassName);
+ try {
+ Constructor<? extends TaskSchedulerService> ctor = taskSchedulerClazz
+ .getConstructor(TaskSchedulerAppCallback.class, AppContext.class, String.class,
+ int.class, String.class, Configuration.class);
+ ctor.setAccessible(true);
+ return ctor.newInstance(this, appContext, host, port, trackingUrl, getConfig());
+ } catch (NoSuchMethodException e) {
+ throw new TezUncheckedException(e);
+ } catch (InvocationTargetException e) {
+ throw new TezUncheckedException(e);
+ } catch (InstantiationException e) {
+ throw new TezUncheckedException(e);
+ } catch (IllegalAccessException e) {
+ throw new TezUncheckedException(e);
}
}
}
+
+ @VisibleForTesting
+ protected void instantiateScheduelrs(String host, int port, String trackingUrl, AppContext appContext) {
+ // Iterate over the list and create all the taskSchedulers
+ for (int i = 0; i < taskSchedulerClasses.length; i++) {
+ taskSchedulers[i] = createTaskScheduler(host, port,
+ trackingUrl, appContext, taskSchedulerClasses[i]);
+ }
+ }
+
@Override
public synchronized void serviceStart() {
@@ -377,13 +398,17 @@ public class TaskSchedulerEventHandler extends AbstractService
// always try to connect to AM and proxy the response. hence it wont work if the webUIService
// is not enabled.
String trackingUrl = (webUI != null) ? webUI.getTrackingURL() : "";
- taskScheduler = createTaskScheduler(serviceAddr.getHostName(),
- serviceAddr.getPort(), trackingUrl, appContext);
- taskScheduler.init(getConfig());
- taskScheduler.start();
+ instantiateScheduelrs(serviceAddr.getHostName(), serviceAddr.getPort(), trackingUrl, appContext);
+
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ taskSchedulers[i].init(getConfig());
+ taskSchedulers[i].start();
+ }
+
+ // TODO TEZ-2118 Start using multiple task schedulers
if (shouldUnregisterFlag.get()) {
// Flag may have been set earlier when task scheduler was not initialized
- taskScheduler.setShouldUnregister();
+ taskSchedulers[0].setShouldUnregister();
}
this.eventHandlingThread = new Thread("TaskSchedulerEventHandlerThread") {
@@ -432,8 +457,8 @@ public class TaskSchedulerEventHandler extends AbstractService
if (eventHandlingThread != null)
eventHandlingThread.interrupt();
}
- if (taskScheduler != null) {
- ((AbstractService)taskScheduler).stop();
+ if (taskSchedulers[0] != null) {
+ ((AbstractService)taskSchedulers[0]).stop();
}
}
@@ -578,7 +603,7 @@ public class TaskSchedulerEventHandler extends AbstractService
public float getProgress() {
// at this point allocate has been called and so node count must be available
// may change after YARN-1722
- int nodeCount = taskScheduler.getClusterNodeCount();
+ int nodeCount = taskSchedulers[0].getClusterNodeCount();
if (nodeCount != cachedNodeCount) {
cachedNodeCount = nodeCount;
sendEvent(new AMNodeEventNodeCountUpdated(cachedNodeCount));
@@ -593,7 +618,7 @@ public class TaskSchedulerEventHandler extends AbstractService
}
public void dagCompleted() {
- taskScheduler.dagComplete();
+ taskSchedulers[0].dagComplete();
}
public void dagSubmitted() {
@@ -603,7 +628,7 @@ public class TaskSchedulerEventHandler extends AbstractService
@Override
public void preemptContainer(ContainerId containerId) {
- taskScheduler.deallocateContainer(containerId);
+ taskSchedulers[0].deallocateContainer(containerId);
// Inform the Containers about completion.
sendEvent(new AMContainerEventCompleted(containerId, ContainerExitStatus.INVALID,
"Container preempted internally", TaskAttemptTerminationCause.INTERNAL_PREEMPTION));
@@ -612,13 +637,13 @@ public class TaskSchedulerEventHandler extends AbstractService
public void setShouldUnregisterFlag() {
LOG.info("TaskScheduler notified that it should unregister from RM");
this.shouldUnregisterFlag.set(true);
- if (this.taskScheduler != null) {
- this.taskScheduler.setShouldUnregister();
+ if (this.taskSchedulers[0] != null) {
+ this.taskSchedulers[0].setShouldUnregister();
}
}
public boolean hasUnregistered() {
- return this.taskScheduler.hasUnregistered();
+ return this.taskSchedulers[0].hasUnregistered();
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index 2913cee..26fc1ab 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -512,10 +512,11 @@ public class MockDAGAppMaster extends DAGAppMaster {
this.handlerConcurrency = handlerConcurrency;
this.numConcurrentContainers = numConcurrentContainers;
}
-
+
// use mock container launcher for tests
@Override
- protected ContainerLauncherRouter createContainerLauncherRouter(final Configuration conf)
+ protected ContainerLauncherRouter createContainerLauncherRouter(final Configuration conf,
+ String[] containerLaunchers)
throws UnknownHostException {
return new ContainerLauncherRouter(containerLauncher);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
index 8816ab5..d98e5d1 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -50,6 +51,8 @@ public class TestTaskAttemptListenerImplTezDag {
@Test(timeout = 5000)
public void testGetTask() throws IOException {
ApplicationId appId = ApplicationId.newInstance(1000, 1);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ Credentials credentials = new Credentials();
AppContext appContext = mock(AppContext.class);
EventHandler eventHandler = mock(EventHandler.class);
DAG dag = mock(DAG.class);
@@ -57,6 +60,8 @@ public class TestTaskAttemptListenerImplTezDag {
Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
doReturn(eventHandler).when(appContext).getEventHandler();
doReturn(dag).when(appContext).getCurrentDAG();
+ doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+ doReturn(credentials).when(appContext).getAppCredentials();
doReturn(appAcls).when(appContext).getApplicationACLs();
doReturn(amContainerMap).when(appContext).getAllContainers();
NodeId nodeId = NodeId.newInstance("localhost", 0);
@@ -68,7 +73,7 @@ public class TestTaskAttemptListenerImplTezDag {
TaskAttemptListenerImpTezDag taskAttemptListener =
new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
- mock(ContainerHeartbeatHandler.class), null);
+ mock(ContainerHeartbeatHandler.class), null, null);
TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator();
TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
@@ -126,6 +131,8 @@ public class TestTaskAttemptListenerImplTezDag {
@Test(timeout = 5000)
public void testGetTaskMultiplePulls() throws IOException {
ApplicationId appId = ApplicationId.newInstance(1000, 1);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ Credentials credentials = new Credentials();
AppContext appContext = mock(AppContext.class);
EventHandler eventHandler = mock(EventHandler.class);
DAG dag = mock(DAG.class);
@@ -133,8 +140,10 @@ public class TestTaskAttemptListenerImplTezDag {
Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
doReturn(eventHandler).when(appContext).getEventHandler();
doReturn(dag).when(appContext).getCurrentDAG();
+ doReturn(credentials).when(appContext).getAppCredentials();
doReturn(appAcls).when(appContext).getApplicationACLs();
doReturn(amContainerMap).when(appContext).getAllContainers();
+ doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
NodeId nodeId = NodeId.newInstance("localhost", 0);
AMContainer amContainer = mock(AMContainer.class);
Container container = mock(Container.class);
@@ -144,7 +153,7 @@ public class TestTaskAttemptListenerImplTezDag {
TaskAttemptListenerImpTezDag taskAttemptListener =
new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
- mock(ContainerHeartbeatHandler.class), null);
+ mock(ContainerHeartbeatHandler.class), null, null);
TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator();
TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
index 89b77a7..54b9adb 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
@@ -1322,7 +1322,7 @@ public class TestContainerReuse {
InputDescriptor.create("inputClassName"), 1)),
Collections.singletonList(new OutputSpec("vertexName",
OutputDescriptor.create("outputClassName"), 1)), null), ta, locationHint,
- priority.getPriority(), containerContext);
+ priority.getPriority(), containerContext, 0, 0, 0);
return lr;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
index af3e40d..291e786 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
@@ -89,13 +89,13 @@ public class TestTaskSchedulerEventHandler {
public MockTaskSchedulerEventHandler(AppContext appContext,
DAGClientServer clientService, EventHandler eventHandler,
ContainerSignatureMatcher containerSignatureMatcher, WebUIService webUI) {
- super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI);
+ super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI, new String[] {});
}
-
+
@Override
- protected TaskSchedulerService createTaskScheduler(String host, int port,
- String trackingUrl, AppContext appContext) {
- return mockTaskScheduler;
+ protected void instantiateScheduelrs(String host, int port, String trackingUrl,
+ AppContext appContext) {
+ taskSchedulers[0] = mockTaskScheduler;
}
@Override
@@ -194,7 +194,7 @@ public class TestTaskSchedulerEventHandler {
when(mockAppContext.getCurrentDAG().getVertex(affVertexName)).thenReturn(affVertex);
Resource resource = Resource.newInstance(100, 1);
AMSchedulerEventTALaunchRequest event = new AMSchedulerEventTALaunchRequest
- (taId, resource, null, mockTaskAttempt, locHint, 3, null);
+ (taId, resource, null, mockTaskAttempt, locHint, 3, null, 0, 0, 0);
schedulerHandler.notify.set(false);
schedulerHandler.handle(event);
synchronized (schedulerHandler.notify) {
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
index 77c98b7..d775300 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
@@ -127,31 +127,29 @@ class TestTaskSchedulerHelpers {
EventHandler eventHandler,
TezAMRMClientAsync<CookieContainerRequest> amrmClientAsync,
ContainerSignatureMatcher containerSignatureMatcher) {
- super(appContext, null, eventHandler, containerSignatureMatcher, null);
+ super(appContext, null, eventHandler, containerSignatureMatcher, null, new String[]{});
this.amrmClientAsync = amrmClientAsync;
this.containerSignatureMatcher = containerSignatureMatcher;
}
@Override
- public TaskSchedulerService createTaskScheduler(String host, int port,
- String trackingUrl, AppContext appContext) {
- return new TaskSchedulerWithDrainableAppCallback(this,
+ public void instantiateScheduelrs(String host, int port, String trackingUrl, AppContext appContext) {
+ taskSchedulers[0] = new TaskSchedulerWithDrainableAppCallback(this,
containerSignatureMatcher, host, port, trackingUrl, amrmClientAsync,
appContext);
}
public TaskSchedulerService getSpyTaskScheduler() {
- return this.taskScheduler;
+ return taskSchedulers[0];
}
@Override
public void serviceStart() {
- TaskSchedulerService taskSchedulerReal = createTaskScheduler("host", 0, "",
- appContext);
+ instantiateScheduelrs("host", 0, "", appContext);
// Init the service so that reuse configuration is picked up.
- ((AbstractService)taskSchedulerReal).init(getConfig());
- ((AbstractService)taskSchedulerReal).start();
- taskScheduler = spy(taskSchedulerReal);
+ ((AbstractService)taskSchedulers[0]).init(getConfig());
+ ((AbstractService)taskSchedulers[0]).start();
+ taskSchedulers[0] = spy(taskSchedulers[0]);
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/0ae9d4a3/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
index a93c1a4..ae7e7f8 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
@@ -45,6 +45,8 @@ public class TestExternalTezServices {
private static final Log LOG = LogFactory.getLog(TestExternalTezServices.class);
+ private static final String EXT_PUSH_ENTITY_NAME = "ExtServiceTestPush";
+
private static MiniTezCluster tezCluster;
private static MiniDFSCluster dfsCluster;
private static MiniTezTestServiceCluster tezTestServiceCluster;
@@ -106,12 +108,17 @@ public class TestExternalTezServices {
remoteFs.mkdirs(stagingDirPath);
// This is currently configured to push tasks into the Service, and then use the standard RPC
confForJobs.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
- confForJobs.set(TezConfiguration.TEZ_AM_TASK_SCHEDULER_CLASS,
- TezTestServiceTaskSchedulerService.class.getName());
- confForJobs.set(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHER_CLASS,
- TezTestServiceNoOpContainerLauncher.class.getName());
- confForJobs.set(TezConfiguration.TEZ_AM_TASK_COMMUNICATOR_CLASS,
- TezTestServiceTaskCommunicatorImpl.class.getName());
+ confForJobs.set(TezConfiguration.TEZ_AM_TASK_SCHEDULERS,
+ EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceTaskSchedulerService.class.getName());
+ confForJobs.set(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHERS,
+ EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceNoOpContainerLauncher.class.getName());
+ confForJobs.set(TezConfiguration.TEZ_AM_TASK_COMMUNICATORS,
+ EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceTaskCommunicatorImpl.class.getName());
+
+ confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_TASK_SCHEDULER_NAME, EXT_PUSH_ENTITY_NAME);
+ confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME, EXT_PUSH_ENTITY_NAME);
+ confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME, EXT_PUSH_ENTITY_NAME);
+
TezConfiguration tezConf = new TezConfiguration(confForJobs);
[06/50] [abbrv] tez git commit: EZ-1482. Fix memory issues for Local
Mode running concurrent tasks (pramachandran)
Posted by ss...@apache.org.
EZ-1482. Fix memory issues for Local Mode running concurrent tasks (pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/19378d5c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/19378d5c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/19378d5c
Branch: refs/heads/TEZ-2003
Commit: 19378d5c2d79006b19140f2c5b9740d04942a3b8
Parents: 11b5843
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Wed Apr 15 21:48:07 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Wed Apr 15 21:48:07 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/dag/app/launcher/LocalContainerLauncher.java | 10 +++++++---
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/19378d5c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 605577e..56f6ea9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-1482. Fix memory issues for Local Mode running concurrent tasks
TEZ-2033. Update TestOrderedWordCount to add processor configs as history text
and use MR configs correctly
TEZ-2318. Tez UI: source and sink page is broken as they are not populated.
http://git-wip-us.apache.org/repos/asf/tez/blob/19378d5c/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 7f5a7b1..5c8aab6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -90,6 +90,7 @@ public class LocalContainerLauncher extends AbstractService implements
private final String workingDirectory;
private final Map<String, String> localEnv = new HashMap<String, String>();
private final ExecutionContext executionContext;
+ private int numExecutors;
private final ConcurrentHashMap<ContainerId, ListenableFuture<TezChild.ContainerExecutionResult>>
runningContainers =
@@ -122,7 +123,7 @@ public class LocalContainerLauncher extends AbstractService implements
@Override
public synchronized void serviceInit(Configuration conf) {
- int numExecutors = conf.getInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS,
+ numExecutors = conf.getInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS,
TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT);
Preconditions.checkState(numExecutors >=1, "Must have at least 1 executor");
ExecutorService rawExecutor = Executors.newFixedThreadPool(numExecutors,
@@ -328,11 +329,14 @@ public class LocalContainerLauncher extends AbstractService implements
containerEnv.putAll(localEnv);
containerEnv.put(Environment.USER.name(), context.getUser());
- // TODO TEZ-1482. Control the memory available based on number of executors
+ long memAvailable;
+ synchronized (this) { // needed to fix findbugs Inconsistent synchronization warning
+ memAvailable = Runtime.getRuntime().maxMemory() / numExecutors;
+ }
TezChild tezChild =
TezChild.newTezChild(defaultConf, null, 0, containerId.toString(), tokenIdentifier,
attemptNumber, localDirs, workingDirectory, containerEnv, "", executionContext, credentials,
- Runtime.getRuntime().maxMemory(), context.getUser());
+ memAvailable, context.getUser());
tezChild.setUmbilical(tezTaskUmbilicalProtocol);
return tezChild;
}
[12/50] [abbrv] tez git commit: TEZ-2329. UI Query on final dag
status performance improvement (jeagles)
Posted by ss...@apache.org.
TEZ-2329. UI Query on final dag status performance improvement (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/bfb34afb
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/bfb34afb
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/bfb34afb
Branch: refs/heads/TEZ-2003
Commit: bfb34afba0edfb254b05037b3b2ab37e3d3e44cf
Parents: d932579
Author: Jonathan Eagles <je...@gmail.com>
Authored: Thu Apr 16 08:57:17 2015 -0500
Committer: Jonathan Eagles <je...@gmail.com>
Committed: Thu Apr 16 08:57:17 2015 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../src/main/webapp/app/scripts/controllers/dags_controller.js | 6 +++++-
tez-ui/src/main/webapp/app/scripts/helpers/misc.js | 4 ++++
3 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/bfb34afb/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 11c7322..a41a724 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -121,6 +121,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2329. UI Query on final dag status performance improvement
TEZ-2287. Deprecate VertexManagerPluginContext.getTaskContainer().
TEZ-1909. Remove need to copy over all events from attempt 1 to attempt 2 dir
TEZ-2061. Tez UI: vertex id column and filter on tasks page should be changed to vertex name
http://git-wip-us.apache.org/repos/asf/tez/blob/bfb34afb/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
index 9968a6a..f6a7013 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
@@ -55,9 +55,13 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
user: this.user_filter
},
secondary: {
- status: this.status_filter
}
}
+ if (App.Helpers.misc.isFinalDagStatus(this.status_filter)) {
+ filters.primary['status'] = this.status_filter;
+ } else {
+ filters.secondary['status'] = this.status_filter;
+ }
this.setFiltersAndLoadEntities(filters);
},
http://git-wip-us.apache.org/repos/asf/tez/blob/bfb34afb/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
index 53149a4..6794e95 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
@@ -90,6 +90,10 @@ App.Helpers.misc = {
'KILLED', 'FAILED', 'ERROR']) != -1;
},
+ isFinalDagStatus: function(status) {
+ return $.inArray(status, ['SUCCEEDED', 'KILLED', 'FAILED', 'ERROR']) != -1;
+ },
+
isValidTaskStatus: function(status) {
return $.inArray(status, ['RUNNING', 'SUCCEEDED', 'FAILED', 'KILLED']) != -1;
},
[17/50] [abbrv] tez git commit: TEZ-2334.
ContainerManagementProtocolProxy modifies IPC timeout conf without making a
copy. (Chang Li via hitesh)
Posted by ss...@apache.org.
TEZ-2334. ContainerManagementProtocolProxy modifies IPC timeout conf without making a copy. (Chang Li via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/57c62f1f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/57c62f1f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/57c62f1f
Branch: refs/heads/TEZ-2003
Commit: 57c62f1f1fc9cd22dc858d3d22d7c13cdc427084
Parents: cbea5ad
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Apr 17 15:04:02 2015 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Apr 17 15:04:02 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/57c62f1f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1888f15..9c86434 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -292,6 +292,7 @@ TEZ-UI CHANGES (TEZ-8):
Release 0.5.4: Unreleased
ALL CHANGES:
+ TEZ-2334. ContainerManagementProtocolProxy modifies IPC timeout conf without making a copy.
TEZ-2317. Event processing backlog can result in task failures for short
tasks
TEZ-2289. ATSHistoryLoggingService can generate ArrayOutOfBoundsException.
http://git-wip-us.apache.org/repos/asf/tez/blob/57c62f1f/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index cb8e262..a1eb2a7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -259,8 +259,9 @@ public class ContainerLauncherImpl extends AbstractService implements
@Override
public void serviceStart() {
+ // pass a copy of config to ContainerManagementProtocolProxy until YARN-3497 is fixed
cmProxy =
- new ContainerManagementProtocolProxy(getConfig());
+ new ContainerManagementProtocolProxy(new Configuration(getConfig()));
ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
"ContainerLauncher #%d").setDaemon(true).build();
[19/50] [abbrv] tez git commit: TEZ-2212. Notify components on DAG
completion. (sseth)
Posted by ss...@apache.org.
TEZ-2212. Notify components on DAG completion. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/44046f8a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/44046f8a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/44046f8a
Branch: refs/heads/TEZ-2003
Commit: 44046f8a4ae7edab466f6c3789772c97955f5786
Parents: db0a50c
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Apr 20 21:05:04 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Apr 20 21:05:04 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/common/AsyncDispatcher.java | 5 +++
.../org/apache/tez/dag/records/TezDAGID.java | 7 ++++
.../tez/dag/records/TezTaskAttemptID.java | 6 +++
.../org/apache/tez/dag/records/TezTaskID.java | 6 +++
.../org/apache/tez/dag/records/TezVertexID.java | 6 +++
.../org/apache/tez/dag/app/DAGAppMaster.java | 39 ++++++++++++++++++++
.../apache/tez/dag/app/TaskAttemptListener.java | 5 +++
.../dag/app/TaskAttemptListenerImpTezDag.java | 16 ++++++++
.../dag/event/DAGAppMasterEventDagCleanup.java | 31 ++++++++++++++++
.../app/dag/event/DAGAppMasterEventType.java | 4 +-
.../tez/dag/app/launcher/ContainerLauncher.java | 5 +++
.../dag/app/launcher/ContainerLauncherImpl.java | 12 ++++++
.../app/launcher/LocalContainerLauncher.java | 9 +++++
.../dag/app/rm/LocalTaskSchedulerService.java | 2 +-
.../dag/app/rm/TaskSchedulerEventHandler.java | 7 +++-
.../tez/dag/app/rm/TaskSchedulerService.java | 2 +-
.../dag/app/rm/YarnTaskSchedulerService.java | 2 +-
.../app/rm/container/AMContainerHelpers.java | 5 +++
.../dag/app/rm/container/AMContainerMap.java | 6 +++
.../tez/dag/app/rm/node/AMNodeTracker.java | 6 +++
.../apache/tez/dag/app/MockDAGAppMaster.java | 11 +++++-
22 files changed, 187 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c44824c..64b3561 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2212. Notify components on DAG completion.
TEZ-2328. Add tez.runtime.sorter.class & rename tez.runtime.sort.threads
to tez.runtime.pipelined.sorter.sort.threads.
TEZ-2333. Enable local fetch optimization by default.
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java
index b751260..5aaa4cf 100644
--- a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java
+++ b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java
@@ -326,4 +326,9 @@ public class AsyncDispatcher extends CompositeService implements Dispatcher {
}
};
}
+
+ @Private
+ public int getQueueSize() {
+ return eventQueue.size();
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
index d12f760..0fe1d44 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
@@ -23,6 +23,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.text.NumberFormat;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import com.google.common.base.Preconditions;
@@ -66,6 +67,12 @@ public class TezDAGID extends TezID {
Preconditions.checkArgument(applicationId != null, "ApplicationID cannot be null");
return dagIdCache.getUnchecked(new TezDAGID(applicationId, id));
}
+
+ @InterfaceAudience.Private
+ public static void clearCache() {
+ dagIdCache.invalidateAll();
+ dagIdCache.cleanUp();
+ }
/**
* Get a DAGID object from given parts.
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java
index 0896b99..296d577 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java
@@ -74,6 +74,12 @@ public class TezTaskAttemptID extends TezID {
return taskAttemptIDCache.getUnchecked(new TezTaskAttemptID(taskID, id));
}
+ @InterfaceAudience.Private
+ public static void clearCache() {
+ taskAttemptIDCache.invalidateAll();
+ taskAttemptIDCache.cleanUp();
+ }
+
private TezTaskAttemptID(TezTaskID taskId, int id) {
super(id);
if(taskId == null) {
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
index 0a7bc4a..b4c7b32 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
@@ -81,6 +81,12 @@ public class TezTaskID extends TezID {
return taskIDCache.getUnchecked(new TezTaskID(vertexID, id));
}
+ @InterfaceAudience.Private
+ public static void clearCache() {
+ taskIDCache.invalidateAll();
+ taskIDCache.cleanUp();
+ }
+
private TezTaskID(TezVertexID vertexID, int id) {
super(id);
Preconditions.checkArgument(vertexID != null, "vertexID cannot be null");
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
index a9b2625..93ef551 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
@@ -83,6 +83,12 @@ public class TezVertexID extends TezID {
return vertexIDCache.getUnchecked(new TezVertexID(dagId, id));
}
+ @InterfaceAudience.Private
+ public static void clearCache() {
+ vertexIDCache.invalidateAll();
+ vertexIDCache.cleanUp();
+ }
+
private TezVertexID(TezDAGID dagId, int id) {
super(id);
this.dagId = dagId;
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 2924b25..3dd9e4c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -57,6 +57,9 @@ import java.util.regex.Pattern;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
+import org.apache.tez.dag.app.dag.event.DAGAppMasterEventDagCleanup;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -661,7 +664,19 @@ public class DAGAppMaster extends AbstractService {
}
if (!state.equals(DAGAppMasterState.ERROR)) {
if (!sessionStopped.get()) {
+ LOG.info("Central Dispatcher queue size after DAG completion, before cleanup: " +
+ dispatcher.getQueueSize());
LOG.info("Waiting for next DAG to be submitted.");
+
+ // Sending this via the event queue, in case there are pending events which need to be
+ // processed. TaskKilled for example, or ContainerCompletions.
+ // The DAG needs to be part of the event, since the dag can get reset when the next
+ // dag is submitted. The next DAG, however, will not start executing till the cleanup
+ // is complete, since execution start is on the same dispatcher.
+ sendEvent(new DAGAppMasterEventDagCleanup(context.getCurrentDAG()));
+
+ // Leaving the taskSchedulerEventHandler here for now. Doesn't generate new events.
+ // However, eventually it needs to be moved out.
this.taskSchedulerEventHandler.dagCompleted();
state = DAGAppMasterState.IDLE;
} else {
@@ -689,6 +704,27 @@ public class DAGAppMaster extends AbstractService {
this.state = DAGAppMasterState.KILLED;
shutdownHandler.shutdown(true);
break;
+ case DAG_CLEANUP:
+ DAGAppMasterEventDagCleanup cleanupEvent = (DAGAppMasterEventDagCleanup) event;
+ LOG.info("Cleaning up DAG: name=" + cleanupEvent.getDag().getName() + ", with id=" +
+ cleanupEvent.getDag().getID());
+ containerLauncher.dagComplete(cleanupEvent.getDag());
+ taskAttemptListener.dagComplete(cleanupEvent.getDag());
+ nodes.dagComplete(cleanupEvent.getDag());
+ containers.dagComplete(cleanupEvent.getDag());
+ TezTaskAttemptID.clearCache();
+ TezTaskID.clearCache();
+ TezVertexID.clearCache();
+ TezDAGID.clearCache();
+ LOG.info("Completed cleanup for DAG: name=" + cleanupEvent.getDag().getName() + ", with id=" +
+ cleanupEvent.getDag().getID());
+ break;
+ case NEW_DAG_SUBMITTED:
+ // Inform sub-components that a new DAG has been submitted.
+ taskSchedulerEventHandler.dagSubmitted();
+ containerLauncher.dagSubmitted();
+ taskAttemptListener.dagSubmitted();
+ break;
default:
throw new TezUncheckedException(
"AppMaster: No handler for event type: " + event.getType());
@@ -2081,6 +2117,9 @@ public class DAGAppMaster extends AbstractService {
// End of creating the job.
((RunningAppContext) context).setDAG(currentDAG);
+ // Send out an event to inform components that a new DAG has been submitted.
+ // Information about this DAG is available via the context.
+ sendEvent(new DAGAppMasterEvent(DAGAppMasterEventType.NEW_DAG_SUBMITTED));
// create a job event for job initialization
DAGEvent initDagEvent = new DAGEvent(currentDAG.getID(), DAGEventType.DAG_INIT);
// Send init to the job (this does NOT trigger job execution)
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
index aeb0cd5..9caa7cf 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
@@ -21,6 +21,7 @@ package org.apache.tez.dag.app;
import java.net.InetSocketAddress;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.rm.container.AMContainerTask;
import org.apache.tez.dag.records.TezTaskAttemptID;
/**
@@ -38,4 +39,8 @@ public interface TaskAttemptListener {
void unregisterTaskAttempt(TezTaskAttemptID attemptID);
+ void dagComplete(DAG dag);
+
+ void dagSubmitted();
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index a01ca39..b64283b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -282,6 +282,22 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
@Override
+ public void dagComplete(DAG dag) {
+ // TODO TEZ-2335. Cleanup TaskHeartbeat handler structures.
+ // TODO TEZ-2345. Also cleanup attemptInfo map, so that any tasks which heartbeat are told to die.
+ // Container structures remain unchanged - since they could be re-used across restarts.
+ // This becomes more relevant when task kills without container kills are allowed.
+
+ // TODO TEZ-2336. Send a signal to containers indicating DAG completion.
+ }
+
+ @Override
+ public void dagSubmitted() {
+ // Nothing to do right now. Indicates that a new DAG has been submitted and
+ // the context has updated information.
+ }
+
+ @Override
public void registerRunningContainer(ContainerId containerId) {
if (LOG.isDebugEnabled()) {
LOG.debug("ContainerId: " + containerId
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventDagCleanup.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventDagCleanup.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventDagCleanup.java
new file mode 100644
index 0000000..2d1a23a
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventDagCleanup.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.event;
+
+import org.apache.tez.dag.app.dag.DAG;
+
+public class DAGAppMasterEventDagCleanup extends DAGAppMasterEvent {
+
+ private final DAG dag;
+
+ public DAGAppMasterEventDagCleanup(DAG dag) {
+ super(DAGAppMasterEventType.DAG_CLEANUP);
+ this.dag = dag;
+ }
+
+ public DAG getDag() {
+ return this.dag;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventType.java
index ee03ef5..5a102a5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventType.java
@@ -22,5 +22,7 @@ public enum DAGAppMasterEventType {
INTERNAL_ERROR,
AM_REBOOT,
DAG_FINISHED,
- SCHEDULING_SERVICE_ERROR
+ SCHEDULING_SERVICE_ERROR,
+ NEW_DAG_SUBMITTED, // Indicates a new dag being submitted, to notify sub-components
+ DAG_CLEANUP
}
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncher.java
index 35cdeda..305eb50 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncher.java
@@ -20,8 +20,13 @@ package org.apache.tez.dag.app.launcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
public interface ContainerLauncher
extends EventHandler<NMCommunicatorEvent> {
+
+ void dagComplete(DAG dag);
+
+ void dagSubmitted();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index a1eb2a7..94889a1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.tez.dag.app.dag.DAG;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -110,6 +111,17 @@ public class ContainerLauncherImpl extends AbstractService implements
}
}
+ @Override
+ public void dagComplete(DAG dag) {
+ // Nothing required at the moment. Containers are shared across DAGs
+ }
+
+ @Override
+ public void dagSubmitted() {
+ // Nothing to do right now. Indicates that a new DAG has been submitted and
+ // the context has updated information.
+ }
+
private static enum ContainerState {
PREP, FAILED, RUNNING, DONE, KILLED_BEFORE_LAUNCH
}
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 5c8aab6..9faf8c0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -43,6 +43,7 @@ import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.tez.dag.app.dag.DAG;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -154,6 +155,14 @@ public class LocalContainerLauncher extends AbstractService implements
callbackExecutor.shutdownNow();
}
+ @Override
+ public void dagComplete(DAG dag) {
+ }
+
+ @Override
+ public void dagSubmitted() {
+ }
+
// Thread to monitor the queue of incoming NMCommunicator events
private class TezSubTaskRunner implements Runnable {
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
index ce01bfb..51d8b9d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
@@ -112,7 +112,7 @@ public class LocalTaskSchedulerService extends TaskSchedulerService {
}
@Override
- public void resetMatchLocalityForAllHeldContainers() {
+ public void dagComplete() {
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 250446d..19db660 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -563,7 +563,12 @@ public class TaskSchedulerEventHandler extends AbstractService
}
public void dagCompleted() {
- taskScheduler.resetMatchLocalityForAllHeldContainers();
+ taskScheduler.dagComplete();
+ }
+
+ public void dagSubmitted() {
+ // Nothing to do right now. Indicates that a new DAG has been submitted and
+ // the context has updated information.
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerService.java
index 096069b..48d5455 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerService.java
@@ -43,7 +43,7 @@ public abstract class TaskSchedulerService extends AbstractService{
public abstract int getClusterNodeCount();
- public abstract void resetMatchLocalityForAllHeldContainers();
+ public abstract void dagComplete();
public abstract Resource getTotalResources();
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
index 66a6f33..44f5484 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
@@ -838,7 +838,7 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
}
@Override
- public synchronized void resetMatchLocalityForAllHeldContainers() {
+ public synchronized void dagComplete() {
for (HeldContainer heldContainer : heldContainers.values()) {
heldContainer.resetLocalityMatchLevel();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 41e814b..470fa56 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -65,6 +65,11 @@ public class AMContainerHelpers {
private static Map<TezDAGID, ContainerLaunchContext> commonContainerSpecs =
new HashMap<TezDAGID, ContainerLaunchContext>();
+ public static void dagComplete(TezDAGID dagId) {
+ synchronized (commonContainerSpecLock) {
+ commonContainerSpecs.remove(dagId);
+ }
+ }
/**
* Create a {@link LocalResource} record with all the given parameters.
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
index 6037a3a..574c38e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
@@ -21,6 +21,7 @@ package org.apache.tez.dag.app.rm.container;
import java.util.Collection;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.tez.dag.app.dag.DAG;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.service.AbstractService;
@@ -74,4 +75,9 @@ public class AMContainerMap extends AbstractService implements EventHandler<AMCo
public Collection<AMContainer> values() {
return containerMap.values();
}
+
+ public void dagComplete(DAG dag){
+ AMContainerHelpers.dagComplete(dag.getID());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java
index a0be825..102cbe9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java
@@ -22,6 +22,7 @@ import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.tez.dag.app.dag.DAG;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -205,4 +206,9 @@ public class AMNodeTracker extends AbstractService implements
public boolean isBlacklistingIgnored() {
return this.ignoreBlacklisting;
}
+
+ public void dagComplete(DAG dag) {
+ // TODO TEZ-2337 Maybe reset failures from previous DAGs
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/44046f8a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index fca15fd..92dfdb5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.tez.dag.app.dag.DAG;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -127,7 +128,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
AtomicBoolean startScheduling = new AtomicBoolean(true);
AtomicBoolean goFlag;
boolean updateProgress = true;
-
+
LinkedBlockingQueue<ContainerData> containersToProcess = new LinkedBlockingQueue<ContainerData>();
Map<TezTaskID, Integer> preemptedTasks = Maps.newConcurrentMap();
@@ -139,6 +140,14 @@ public class MockDAGAppMaster extends DAGAppMaster {
this.goFlag = goFlag;
}
+ @Override
+ public void dagComplete(DAG dag) {
+ }
+
+ @Override
+ public void dagSubmitted() {
+ }
+
public class ContainerData {
ContainerId cId;
TezTaskAttemptID taId;
[47/50] [abbrv] tez git commit: TEZ-2285. Allow TaskCommunicators to
indicate task/container liveness. (sseth)
Posted by ss...@apache.org.
TEZ-2285. Allow TaskCommunicators to indicate task/container liveness. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/7eca3b75
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/7eca3b75
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/7eca3b75
Branch: refs/heads/TEZ-2003
Commit: 7eca3b752bad3c9c6123c38b1d2d38cfea8409cb
Parents: a9f4da1
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Apr 7 13:22:09 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:48:02 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../apache/tez/dag/api/TaskCommunicatorContext.java | 4 ++++
.../tez/dag/app/TaskAttemptListenerImpTezDag.java | 10 ++++++++++
.../apache/tez/dag/app/TezTaskCommunicatorImpl.java | 16 +++++++++-------
4 files changed, 24 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/7eca3b75/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index e2c428d..9d6b220 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -14,5 +14,6 @@ ALL CHANGES:
TEZ-2241. Miscellaneous fixes after last reabse.
TEZ-2283. Fixes after rebase 04/07.
TEZ-2284. Separate TaskReporter into an interface.
+ TEZ-2285. Allow TaskCommunicators to indicate task/container liveness.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/7eca3b75/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
index a85fb7f..0c3bac3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
@@ -43,6 +43,10 @@ public interface TaskCommunicatorContext {
boolean isKnownContainer(ContainerId containerId);
+ void taskAlive(TezTaskAttemptID taskAttemptId);
+
+ void containerAlive(ContainerId containerId);
+
// TODO TEZ-2003 Move to vertex, taskIndex, version. Rename to taskAttempt*
void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId);
http://git-wip-us.apache.org/repos/asf/tez/blob/7eca3b75/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index bea0902..0a6710c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -236,6 +236,16 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
@Override
+ public void taskAlive(TezTaskAttemptID taskAttemptId) {
+ taskHeartbeatHandler.pinged(taskAttemptId);
+ }
+
+ @Override
+ public void containerAlive(ContainerId containerId) {
+ pingContainerHeartbeatHandler(containerId);
+ }
+
+ @Override
public void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId) {
context.getEventHandler()
.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, containerId, null));
http://git-wip-us.apache.org/repos/asf/tez/blob/7eca3b75/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
index 9d0c031..ef4f764 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
@@ -65,17 +65,19 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
null, true, null, null, false);
private final TaskCommunicatorContext taskCommunicatorContext;
+ private final TezTaskUmbilicalProtocol taskUmbilical;
- private final ConcurrentMap<ContainerId, ContainerInfo> registeredContainers =
+ protected final ConcurrentMap<ContainerId, ContainerInfo> registeredContainers =
new ConcurrentHashMap<ContainerId, ContainerInfo>();
- private final ConcurrentMap<TaskAttempt, ContainerId> attemptToContainerMap =
+ protected final ConcurrentMap<TaskAttempt, ContainerId> attemptToContainerMap =
new ConcurrentHashMap<TaskAttempt, ContainerId>();
- private final TezTaskUmbilicalProtocol taskUmbilical;
- private final String tokenIdentifier;
- private final Token<JobTokenIdentifier> sessionToken;
+
+ protected final String tokenIdentifier;
+ protected final Token<JobTokenIdentifier> sessionToken;
protected InetSocketAddress address;
- private Server server;
+
+ protected volatile Server server;
public static final class ContainerInfo {
@@ -438,7 +440,7 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
// Holder for Task information, which eventually will likely be VertexImplm taskIndex, attemptIndex
- private static class TaskAttempt {
+ protected static class TaskAttempt {
// TODO TEZ-2003 Change this to work with VertexName, int id, int version
// TODO TEZ-2003 Avoid constructing this unit all over the place
private TezTaskAttemptID taskAttemptId;
[05/50] [abbrv] tez git commit: TEZ-2287. Deprecate
VertexManagerPluginContext.getTaskContainer(). (hitesh)
Posted by ss...@apache.org.
TEZ-2287. Deprecate VertexManagerPluginContext.getTaskContainer(). (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/11b58431
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/11b58431
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/11b58431
Branch: refs/heads/TEZ-2003
Commit: 11b584318762d45ce5173eebde9a4c7a86c3d979
Parents: beb7785
Author: Hitesh Shah <hi...@apache.org>
Authored: Tue Apr 14 11:55:04 2015 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Tue Apr 14 11:55:04 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/tez/dag/api/VertexManagerPluginContext.java | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/11b58431/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cdc48e1..605577e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -118,6 +118,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2287. Deprecate VertexManagerPluginContext.getTaskContainer().
TEZ-1909. Remove need to copy over all events from attempt 1 to attempt 2 dir
TEZ-2061. Tez UI: vertex id column and filter on tasks page should be changed to vertex name
TEZ-2242. Refactor ShuffleVertexManager code
http://git-wip-us.apache.org/repos/asf/tez/blob/11b58431/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
index ab4ced0..afcdb88 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
@@ -118,6 +118,7 @@ public interface VertexManagerPluginContext {
* @return YARN container for the successful task. Maybe null if there is no
* successful task.
*/
+ @Deprecated
public Container getTaskContainer(String vertexName, Integer taskIndex);
/**
[44/50] [abbrv] tez git commit: TEZ-2175. Task priority should be
available to the TaskCommunicator plugin. (sseth)
Posted by ss...@apache.org.
TEZ-2175. Task priority should be available to the TaskCommunicator plugin. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/6fa456ad
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/6fa456ad
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/6fa456ad
Branch: refs/heads/TEZ-2003
Commit: 6fa456ad397321237bd4938bac48187aad47057c
Parents: e8f804f
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Mar 10 00:47:07 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:47:33 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../src/main/java/org/apache/tez/dag/api/TaskCommunicator.java | 2 +-
.../org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java | 2 +-
.../java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java | 3 ++-
.../dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java | 5 +++--
5 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/6fa456ad/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 5c5fd8e..7726815 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -9,5 +9,6 @@ ALL CHANGES:
TEZ-2131. Add additional tests for tasks running in the AM.
TEZ-2138. Fix minor bugs in adding default scheduler, getting launchers.
TEZ-2139. Update tez version to 0.7.0-TEZ-2003-SNAPSHOT.
+ TEZ-2175. Task priority should be available to the TaskCommunicator plugin.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/6fa456ad/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
index c9f85e0..82eed20 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
@@ -43,7 +43,7 @@ public abstract class TaskCommunicator extends AbstractService {
public abstract void registerRunningTaskAttempt(ContainerId containerId, TaskSpec taskSpec,
Map<String, LocalResource> additionalResources,
Credentials credentials,
- boolean credentialsChanged);
+ boolean credentialsChanged, int priority);
// TODO TEZ-2003 Remove reference to TaskAttemptID
public abstract void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID);
http://git-wip-us.apache.org/repos/asf/tez/blob/6fa456ad/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 5435633..ee5d4ee 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -337,7 +337,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
taskCommunicators[taskCommId].registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
amContainerTask.getAdditionalResources(), amContainerTask.getCredentials(),
- amContainerTask.haveCredentialsChanged());
+ amContainerTask.haveCredentialsChanged(), amContainerTask.getPriority());
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/6fa456ad/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
index 0bf1b5d..f288748 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
@@ -194,7 +194,8 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
@Override
public void registerRunningTaskAttempt(ContainerId containerId, TaskSpec taskSpec,
Map<String, LocalResource> additionalResources,
- Credentials credentials, boolean credentialsChanged) {
+ Credentials credentials, boolean credentialsChanged,
+ int priority) {
ContainerInfo containerInfo = registeredContainers.get(containerId);
Preconditions.checkNotNull(containerInfo,
http://git-wip-us.apache.org/repos/asf/tez/blob/6fa456ad/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
index 78cdcde..a327caf 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
@@ -100,9 +100,10 @@ public class TezTestServiceTaskCommunicatorImpl extends TezTaskCommunicatorImpl
public void registerRunningTaskAttempt(final ContainerId containerId, final TaskSpec taskSpec,
Map<String, LocalResource> additionalResources,
Credentials credentials,
- boolean credentialsChanged) {
+ boolean credentialsChanged,
+ int priority) {
super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials,
- credentialsChanged);
+ credentialsChanged, priority);
SubmitWorkRequestProto requestProto = null;
try {
requestProto = constructSubmitWorkRequest(containerId, taskSpec);
[02/50] [abbrv] tez git commit: TEZ-2016. Tez UI: Dag View Fit and
Finish (Sreenath Somarajapuram via pramachandran)
Posted by ss...@apache.org.
TEZ-2016. Tez UI: Dag View Fit and Finish (Sreenath Somarajapuram via pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/6019bbcc
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/6019bbcc
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/6019bbcc
Branch: refs/heads/TEZ-2003
Commit: 6019bbccfcae4e217b4331a25c7c822c164a34c6
Parents: cb6721b
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Mon Apr 13 22:46:57 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Mon Apr 13 22:46:57 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../scripts/components/dag-view/graph-view.js | 86 +++++++++++++++++---
.../app/scripts/components/dag-view/tip.js | 2 +-
.../main/webapp/app/scripts/helpers/dialogs.js | 2 +-
tez-ui/src/main/webapp/app/styles/dag-view.less | 32 +++++++-
.../app/templates/components/dag-view.hbs | 10 ++-
.../src/main/webapp/app/templates/dag/view.hbs | 2 +-
7 files changed, 111 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/6019bbcc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index df8b06c..5fe83cb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2016. Tez UI: Dag View Fit and Finish
TEZ-2252. Tez UI: in graphical view some of the sinks are hidden as they overlap
TEZ-2275. Tez UI: enable faster loading and caching of data in tables
TEZ-2234. Add API for statistics information - allow vertex managers to get
http://git-wip-us.apache.org/repos/asf/tez/blob/6019bbcc/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js b/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
index 6b54ce4..45202ae 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
@@ -132,6 +132,8 @@ App.DagViewComponent.graphView = (function (){
_scheduledClickId = 0, // Id of scheduled click, used for double click.
_tip, // Instance of tip.js
+
+ _panZoomValues, // Temporary storage of pan zoom values for fit toggle
_panZoom; // A closure returned by _attachPanZoom to reset/modify pan and zoom values
function _getName(d) {
@@ -250,7 +252,7 @@ App.DagViewComponent.graphView = (function (){
node.append('use').attr('xlink:href', '#%@-bg'.fmt(className));
node.append('text')
.attr('class', 'title')
- .text(_trimText(d.get(titleProperty || 'name'), maxTitleLength || 3));
+ .text(_trimText(d.get(titleProperty || 'name'), maxTitleLength || 12));
}
/**
* Populates the calling node with the required content.
@@ -261,7 +263,7 @@ App.DagViewComponent.graphView = (function (){
switch(d.type) {
case 'vertex':
- _addBasicContents(node, d, 'vertexName', 15);
+ _addBasicContents(node, d, 'vertexName');
_addStatusBar(node, d);
_addTaskBubble(node, d);
_addIOBubble(node, d);
@@ -376,6 +378,8 @@ App.DagViewComponent.graphView = (function (){
value = property.getCellContent(d.get('data'));
}
+ value = App.Helpers.number.formatNumThousands(value);
+
if(typeof value != 'object') {
list[property.get('headerCellName')] = value;
}
@@ -390,7 +394,7 @@ App.DagViewComponent.graphView = (function (){
var list = {
"Class": App.Helpers.misc.getClassName(d.get("class")),
"Initializer": App.Helpers.misc.getClassName(d.get("initializer")),
- "Configurations": d.get("configs.length")
+ "Configurations": App.Helpers.number.formatNumThousands(d.get("configs.length"))
};
tooltipData = {
title: d.get("name"),
@@ -400,7 +404,7 @@ App.DagViewComponent.graphView = (function (){
case "output":
var list = {
"Class": App.Helpers.misc.getClassName(d.get("class")),
- "Configurations": d.get("configs.length")
+ "Configurations": App.Helpers.number.formatNumThousands(d.get("configs.length"))
};
tooltipData = {
title: d.get("name"),
@@ -446,8 +450,8 @@ App.DagViewComponent.graphView = (function (){
"Data Movement Type": d.get("dataMovementType"),
"Data Source Type": d.get("dataSourceType"),
"Scheduling Type": d.get("schedulingType"),
- "Edge Destination Class": App.Helpers.misc.getClassName(d.get("edgeDestinationClass")),
- "Edge Source Class": App.Helpers.misc.getClassName(d.get("edgeSourceClass"))
+ "Edge Source Class": App.Helpers.misc.getClassName(d.get("edgeSourceClass")),
+ "Edge Destination Class": App.Helpers.misc.getClassName(d.get("edgeDestinationClass"))
};
}
else {
@@ -713,7 +717,9 @@ App.DagViewComponent.graphView = (function (){
panX = PADDING,
panY = PADDING,
- scale = 1;
+ scale = 1,
+
+ scheduleId = 0;
/**
* Transform g to current panX, panY and scale.
@@ -725,6 +731,33 @@ App.DagViewComponent.graphView = (function (){
}
/**
+ * Check if the item have moved out of the visible area, and reset if required
+ */
+ function visibilityCheck() {
+ var graphBound = g.node().getBoundingClientRect(),
+ containerBound = container[0].getBoundingClientRect();
+
+ if(graphBound.right < containerBound.left ||
+ graphBound.bottom < containerBound.top ||
+ graphBound.left > containerBound.right ||
+ graphBound.top > containerBound.bottom) {
+ panX = PADDING, panY = PADDING, scale = 1;
+ transform(true);
+ }
+ }
+
+ /**
+ * Schedule a visibility check and reset if required
+ */
+ function scheduleVisibilityCheck() {
+ if(scheduleId) {
+ clearTimeout(scheduleId);
+ scheduleId = 0;
+ }
+ scheduleId = setTimeout(visibilityCheck, 100);
+ }
+
+ /**
* Set pan values
*/
function onMouseMove(event) {
@@ -760,6 +793,7 @@ App.DagViewComponent.graphView = (function (){
panY += (mouseY - panY) * factor;
transform();
+ scheduleVisibilityCheck();
_tip.reposition();
event.preventDefault();
@@ -772,9 +806,13 @@ App.DagViewComponent.graphView = (function (){
prevY = event.pageY;
container.on('mousemove', onMouseMove);
+ container.parent().addClass('panning');
})
.mouseup(function (event){
container.off('mousemove', onMouseMove);
+ container.parent().removeClass('panning');
+
+ scheduleVisibilityCheck();
})
/**
@@ -783,12 +821,20 @@ App.DagViewComponent.graphView = (function (){
* @param newPanY {Number}
* @param newScale {Number}
*/
- return function (newPanX, newPanY, newScale) {
- panX = newPanX,
- panY = newPanY,
- scale = newScale;
+ return function(newPanX, newPanY, newScale) {
+ var values = {
+ panX: panX,
+ panY: panY,
+ scale: scale
+ };
+
+ panX = newPanX == undefined ? panX : newPanX,
+ panY = newPanY == undefined ? panY : newPanY,
+ scale = newScale == undefined ? scale : newScale;
transform(true);
+
+ return values;
}
}
@@ -851,8 +897,22 @@ App.DagViewComponent.graphView = (function (){
var scale = Math.min(
(_svg.width() - PADDING * 2) / _width,
(_svg.height() - PADDING * 2) / _height
- );
- _panZoom(PADDING, PADDING, scale);
+ ),
+ panZoomValues = _panZoom();
+
+ if(
+ panZoomValues.panX != PADDING ||
+ panZoomValues.panY != PADDING ||
+ panZoomValues.scale != scale
+ ) {
+ _panZoomValues = _panZoom(PADDING, PADDING, scale);
+ }
+ else {
+ _panZoomValues = _panZoom(
+ _panZoomValues.panX,
+ _panZoomValues.panY,
+ _panZoomValues.scale);
+ }
},
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/6019bbcc/tez-ui/src/main/webapp/app/scripts/components/dag-view/tip.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/dag-view/tip.js b/tez-ui/src/main/webapp/app/scripts/components/dag-view/tip.js
index fa95a66..e26223d 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/dag-view/tip.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/dag-view/tip.js
@@ -51,7 +51,7 @@ App.DagViewComponent.tip = (function () {
"<tr><td>",
property,
"</td><td>",
- App.Helpers.number.formatNumThousands(value),
+ value,
"</td></tr>"
);
});
http://git-wip-us.apache.org/repos/asf/tez/blob/6019bbcc/tez-ui/src/main/webapp/app/scripts/helpers/dialogs.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/dialogs.js b/tez-ui/src/main/webapp/app/scripts/helpers/dialogs.js
index abfc62b..ddf7535 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/dialogs.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/dialogs.js
@@ -35,7 +35,7 @@ App.Dialogs = Em.Namespace.create({
}
var container = $( "<div/>" ),
- listHTML = "";
+ listHTML = "<input type='hidden' autofocus='autofocus'/>";
listItems.forEach(function (item) {
var id = getProperty(item, 'id'),
http://git-wip-us.apache.org/repos/asf/tez/blob/6019bbcc/tez-ui/src/main/webapp/app/styles/dag-view.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/dag-view.less b/tez-ui/src/main/webapp/app/styles/dag-view.less
index e51c6e8..0bf0070 100644
--- a/tez-ui/src/main/webapp/app/styles/dag-view.less
+++ b/tez-ui/src/main/webapp/app/styles/dag-view.less
@@ -40,6 +40,16 @@
overflow:hidden;
height: 100%;
+ cursor: -moz-grab;
+ cursor: -webkit-grab;
+ cursor: grab;
+
+ &.panning {
+ cursor: -moz-grabbing;
+ cursor: -webkit-grabbing;
+ cursor: grabbing;
+ }
+
svg {
width: 100%;
height: 100%;
@@ -156,7 +166,7 @@
.no-select;
pointer-events: none;
- font: 12px sans-serif;
+ font: 11px sans-serif;
text-anchor: middle;
-webkit-transform: translate(0px, 4px); // For safari
@@ -216,13 +226,21 @@
.task-status {
-webkit-animation: none !important;
- font-size: 12px;
+ font-size: 11px;
margin-top: 3px;
background-color: rgba(255, 255, 255, 0.8);
border-radius: 6px;
+ vertical-align: -1px;
+
+ width: 8px;
+ height: 7px;
- width: 10px;
- height: 8px;
+ &.running {
+ font-size: 11px;
+
+ width: 9px;
+ height: 7px
+ }
}
}
}
@@ -243,6 +261,8 @@
stroke: #ccc;
stroke-width: 3px;
+ cursor: default;
+
&.broadcast {
stroke: #ccbb8f;
}
@@ -278,6 +298,9 @@
font-size: 1.1em;
}
.tip-text {
+ border-top: 1px solid rgba(255, 255, 255, 0.4);
+ text-align: center;
+ margin-bottom: -1px;
}
.tip-list {
table {
@@ -288,6 +311,7 @@
td {
overflow: hidden;
white-space: nowrap;
+ text-overflow: ellipsis;
max-width: 400px;
}
td:nth-child(1) {
http://git-wip-us.apache.org/repos/asf/tez/blob/6019bbcc/tez-ui/src/main/webapp/app/templates/components/dag-view.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/dag-view.hbs b/tez-ui/src/main/webapp/app/templates/components/dag-view.hbs
index a41f772..eb9c661 100644
--- a/tez-ui/src/main/webapp/app/templates/components/dag-view.hbs
+++ b/tez-ui/src/main/webapp/app/templates/components/dag-view.hbs
@@ -28,12 +28,14 @@
class="vertex-node-bg"
style="fill: url(#vertex-grad); filter: url(#grey-glow)"
rx="5" ry="5" width="80" height="30" x="-40" y="-15"/>
- <circle id="input-bg" class="input-node-bg"
+ <rect id="input-bg"
+ class="input-node-bg"
style="fill: url(#input-grad); filter: url(#grey-glow)"
- cx="0" cy="0" r="15"/>
- <circle id="output-bg" class="output-node-bg"
+ rx="15" ry="15" width="80" height="30" x="-40" y="-15"/>
+ <rect id="output-bg"
+ class="output-node-bg"
style="fill: url(#output-grad); filter: url(#grey-glow)"
- cx="0" cy="0" r="15"/>
+ rx="15" ry="15" width="80" height="30" x="-40" y="-15"/>
<circle id="task-bubble" class="task-bubble-bg"
style="fill: url(#task-grad); filter: url(#grey-glow)"
r="10"/>
http://git-wip-us.apache.org/repos/asf/tez/blob/6019bbcc/tez-ui/src/main/webapp/app/templates/dag/view.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/dag/view.hbs b/tez-ui/src/main/webapp/app/templates/dag/view.hbs
index 699d0c8..3ea27c7 100644
--- a/tez-ui/src/main/webapp/app/templates/dag/view.hbs
+++ b/tez-ui/src/main/webapp/app/templates/dag/view.hbs
@@ -31,7 +31,7 @@
entityClicked='entityClicked'
configure='selectColumns'
}}
- <div class="dag-view-legend">Double click source/sink bubble to toggle visibility locally.</div>
+ <div class="dag-view-legend">When sources & sinks are hidden, double click green bubble to toggle visibility locally.</div>
{{else}}
{{partial 'partials/loading-spinner'}}
{{/unless}}
[09/50] [abbrv] tez git commit: TEZ-714. OutputCommitters should not
run in the main AM dispatcher thread (zjffdu)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
new file mode 100644
index 0000000..0df8a4f
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
@@ -0,0 +1,1682 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.dag.app.dag.impl;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tez.common.security.ACLManager;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.DataSinkDescriptor;
+import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
+import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
+import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
+import org.apache.tez.dag.api.GroupInputEdge;
+import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.OutputCommitterDescriptor;
+import org.apache.tez.dag.api.OutputDescriptor;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.VertexManagerPlugin;
+import org.apache.tez.dag.api.VertexManagerPluginContext;
+import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.oldrecords.TaskState;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskHeartbeatHandler;
+import org.apache.tez.dag.app.dag.DAGState;
+import org.apache.tez.dag.app.dag.DAGTerminationCause;
+import org.apache.tez.dag.app.dag.Task;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.VertexState;
+import org.apache.tez.dag.app.dag.VertexTerminationCause;
+import org.apache.tez.dag.app.dag.event.CallableEventType;
+import org.apache.tez.dag.app.dag.event.DAGAppMasterEventDAGFinished;
+import org.apache.tez.dag.app.dag.event.DAGAppMasterEventType;
+import org.apache.tez.dag.app.dag.event.DAGEvent;
+import org.apache.tez.dag.app.dag.event.DAGEventStartDag;
+import org.apache.tez.dag.app.dag.event.DAGEventType;
+import org.apache.tez.dag.app.dag.event.DAGEventVertexReRunning;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
+import org.apache.tez.dag.app.dag.event.TaskEvent;
+import org.apache.tez.dag.app.dag.event.TaskEventType;
+import org.apache.tez.dag.app.dag.event.VertexEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted;
+import org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule;
+import org.apache.tez.dag.app.dag.event.VertexEventType;
+import org.apache.tez.dag.app.dag.impl.DAGImpl.OutputKey;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.HistoryEventHandler;
+import org.apache.tez.dag.history.HistoryEventType;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.OutputCommitter;
+import org.apache.tez.runtime.api.OutputCommitterContext;
+import org.apache.tez.runtime.api.events.VertexManagerEvent;
+import org.apache.tez.runtime.api.impl.EventMetaData;
+import org.apache.tez.runtime.api.impl.TezEvent;
+import org.apache.tez.runtime.api.impl.EventMetaData.EventProducerConsumerType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ *
+ * The test case of commit here are different from that in TestDAGImpl &
+ * TestVertexImpl in that the commits here are running in separated thread. So
+ * should need to pay some special attention.
+ *
+ * 2 kinds of commit
+ * <li> test XXX_OnDAGSuccess means TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS is true
+ * <li> test XXX_OnVertexSuccess means TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS is false
+ *
+ */
+public class TestCommit {
+
+ private static final Log LOG = LogFactory.getLog(TestCommit.class);
+ private TezDAGID dagId;
+ private static Configuration conf = new Configuration();
+ private DrainDispatcher dispatcher;
+ private Credentials fsTokens;
+ private AppContext appContext;
+ private ACLManager aclManager;
+ private ApplicationAttemptId appAttemptId;
+ private DAGImpl dag;
+ private TaskEventDispatcher taskEventDispatcher;
+ private VertexEventDispatcher vertexEventDispatcher;
+ private DagEventDispatcher dagEventDispatcher;
+ private TaskAttemptListener taskAttemptListener;
+ private TaskHeartbeatHandler thh;
+ private Clock clock = new SystemClock();
+ private DAGFinishEventHandler dagFinishEventHandler;
+ private HistoryEventHandler historyEventHandler;
+ private TaskAttemptEventDispatcher taskAttemptEventDispatcher;
+
+ private ExecutorService rawExecutor;
+ private ListeningExecutorService execService;
+
+ private class DagEventDispatcher implements EventHandler<DAGEvent> {
+ @Override
+ public void handle(DAGEvent event) {
+ dag.handle(event);
+ }
+ }
+
+ private class VertexEventDispatcher implements EventHandler<VertexEvent> {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void handle(VertexEvent event) {
+ Vertex vertex = dag.getVertex(event.getVertexId());
+ ((EventHandler<VertexEvent>) vertex).handle(event);
+ }
+ }
+
+ private class TaskEventDispatcher implements EventHandler<TaskEvent> {
+ @SuppressWarnings("unchecked")
+ @Override
+ public void handle(TaskEvent event) {
+ Vertex vertex = dag.getVertex(event.getTaskID().getVertexID());
+ Task task = vertex.getTask(event.getTaskID());
+ ((EventHandler<TaskEvent>) task).handle(event);
+ }
+ }
+
+ private class TaskAttemptEventDispatcher implements
+ EventHandler<TaskAttemptEvent> {
+ @Override
+ public void handle(TaskAttemptEvent event) {
+ // Ignore
+ }
+ }
+
+ private class DAGFinishEventHandler implements
+ EventHandler<DAGAppMasterEventDAGFinished> {
+ @Override
+ public void handle(DAGAppMasterEventDAGFinished event) {
+ }
+ }
+
+ public static class CountingOutputCommitter extends OutputCommitter {
+
+ public volatile int initCounter = 0;
+ public volatile int setupCounter = 0;
+ public volatile int commitCounter = 0;
+ public volatile int abortCounter = 0;
+ private boolean throwError;
+ private volatile boolean blockCommit;
+
+ public CountingOutputCommitter(OutputCommitterContext context) {
+ super(context);
+ this.throwError = false;
+ }
+
+ @Override
+ public void initialize() throws IOException {
+ if (getContext().getUserPayload() != null
+ && getContext().getUserPayload().hasPayload()) {
+ CountingOutputCommitterConfig conf = new CountingOutputCommitterConfig(
+ getContext().getUserPayload());
+ this.throwError = conf.throwError;
+ this.blockCommit = conf.blockCommit;
+ }
+ ++initCounter;
+ }
+
+ @Override
+ public void setupOutput() throws IOException {
+ ++setupCounter;
+ }
+
+ @Override
+ public void commitOutput() throws IOException {
+ ++commitCounter;
+ while (blockCommit) {
+ try {
+ Thread.sleep(100);
+ LOG.info("committing output:" + getContext().getOutputName());
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+ if (throwError) {
+ throw new RuntimeException("I can throwz exceptions in commit");
+ }
+ }
+
+ public void unblockCommit() {
+ blockCommit = false;
+ }
+
+ @Override
+ public void abortOutput(VertexStatus.State finalState) throws IOException {
+ ++abortCounter;
+ }
+
+ public static class CountingOutputCommitterConfig implements Writable {
+
+ boolean throwError = false;
+ boolean blockCommit = false;
+
+ public CountingOutputCommitterConfig() {
+ }
+
+ public CountingOutputCommitterConfig(boolean throwError,
+ boolean blockCommit) {
+ this.throwError = throwError;
+ this.blockCommit = blockCommit;
+ }
+
+ public CountingOutputCommitterConfig(UserPayload payload)
+ throws IOException {
+ DataInputByteBuffer in = new DataInputByteBuffer();
+ in.reset(payload.getPayload());
+ this.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeBoolean(throwError);
+ out.writeBoolean(blockCommit);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ throwError = in.readBoolean();
+ blockCommit = in.readBoolean();
+ }
+
+ public byte[] toUserPayload() throws IOException {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutput out = new DataOutputStream(bos);
+ write(out);
+ return bos.toByteArray();
+ }
+ }
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ public void setupDAG(DAGPlan dagPlan) {
+ conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
+ appAttemptId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(100, 1), 1);
+ dagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 1);
+ Assert.assertNotNull(dagId);
+ dispatcher = new DrainDispatcher();
+ fsTokens = new Credentials();
+ appContext = mock(AppContext.class);
+ rawExecutor = Executors.newCachedThreadPool(new ThreadFactoryBuilder()
+ .setDaemon(true).setNameFormat("App Shared Pool - " + "#%d").build());
+ execService = MoreExecutors.listeningDecorator(rawExecutor);
+
+ doReturn(execService).when(appContext).getExecService();
+ historyEventHandler = mock(HistoryEventHandler.class);
+ aclManager = new ACLManager("amUser");
+ doReturn(conf).when(appContext).getAMConf();
+ doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+ doReturn(appAttemptId.getApplicationId()).when(appContext)
+ .getApplicationID();
+ doReturn(dagId).when(appContext).getCurrentDAGID();
+ doReturn(historyEventHandler).when(appContext).getHistoryHandler();
+ doReturn(aclManager).when(appContext).getAMACLManager();
+ dag = new DAGImpl(dagId, conf, dagPlan, dispatcher.getEventHandler(),
+ taskAttemptListener, fsTokens, clock, "user", thh, appContext);
+ doReturn(dag).when(appContext).getCurrentDAG();
+ doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
+
+ dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
+ taskEventDispatcher = new TaskEventDispatcher();
+ dispatcher.register(TaskEventType.class, taskEventDispatcher);
+ taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
+ dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
+ vertexEventDispatcher = new VertexEventDispatcher();
+ dispatcher.register(VertexEventType.class, vertexEventDispatcher);
+ dagEventDispatcher = new DagEventDispatcher();
+ dispatcher.register(DAGEventType.class, dagEventDispatcher);
+ dagFinishEventHandler = new DAGFinishEventHandler();
+ dispatcher.register(DAGAppMasterEventType.class, dagFinishEventHandler);
+ dispatcher.init(conf);
+ dispatcher.start();
+ }
+
+ @After
+ public void teardown() {
+ if (dispatcher != null) {
+ dispatcher.await();
+ dispatcher.stop();
+ }
+ if (execService != null) {
+ execService.shutdownNow();
+ }
+ }
+
+ private void waitUntil(DAGImpl dag, DAGState state) {
+ while (dag.getState() != state) {
+ LOG.info("Wait for dag go to state:" + state);
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private void waitUntil(VertexImpl vertex, VertexState state) {
+ while (vertex.getState() != state) {
+ LOG.info("Wait for vertex " + vertex.getLogIdentifier() + " go to state:"
+ + state);
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private void waitForCommitCompleted(VertexImpl vertex, String outputName) {
+ while (vertex.commitFutures.containsKey(outputName)) {
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ LOG.info("Wait for vertex commit " + outputName + " to complete");
+ }
+ }
+
+ private void waitForCommitCompleted(DAGImpl vertex, OutputKey outputKey) {
+ while (vertex.commitFutures.containsKey(outputKey)) {
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ LOG.info("Wait for dag commit " + outputKey + " to complete");
+ }
+ }
+
+ // v1->v3
+ // v2->v3
+ // vertex_group (v1, v2)
+ private DAGPlan createDAGPlan(boolean vertexGroupCommitSucceeded,
+ boolean v3CommitSucceeded) throws Exception {
+ LOG.info("Setting up group dag plan");
+ int dummyTaskCount = 1;
+ Resource dummyTaskResource = Resource.newInstance(1, 1);
+ org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create(
+ "vertex1", ProcessorDescriptor.create("Processor"), dummyTaskCount,
+ dummyTaskResource);
+ org.apache.tez.dag.api.Vertex v2 = org.apache.tez.dag.api.Vertex.create(
+ "vertex2", ProcessorDescriptor.create("Processor"), dummyTaskCount,
+ dummyTaskResource);
+ org.apache.tez.dag.api.Vertex v3 = org.apache.tez.dag.api.Vertex.create(
+ "vertex3", ProcessorDescriptor.create("Processor"), dummyTaskCount,
+ dummyTaskResource);
+
+ DAG dag = DAG.create("testDag");
+ String groupName1 = "uv12";
+ OutputCommitterDescriptor ocd1 = OutputCommitterDescriptor.create(
+ CountingOutputCommitter.class.getName()).setUserPayload(
+ UserPayload.create(ByteBuffer
+ .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig(
+ !vertexGroupCommitSucceeded, true).toUserPayload())));
+ OutputCommitterDescriptor ocd2 = OutputCommitterDescriptor.create(
+ CountingOutputCommitter.class.getName()).setUserPayload(
+ UserPayload.create(ByteBuffer
+ .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig(
+ !v3CommitSucceeded, true).toUserPayload())));
+
+ org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1,
+ v1, v2);
+ OutputDescriptor outDesc = OutputDescriptor.create("output.class");
+ uv12.addDataSink("v12Out", DataSinkDescriptor.create(outDesc, ocd1, null));
+ v3.addDataSink("v3Out", DataSinkDescriptor.create(outDesc, ocd2, null));
+
+ GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(
+ DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")), InputDescriptor
+ .create("merge.class"));
+
+ dag.addVertex(v1);
+ dag.addVertex(v2);
+ dag.addVertex(v3);
+ dag.addEdge(e1);
+ return dag.createDag(conf, null, null, null, true);
+ }
+
+ private DAGPlan createDAGPlan_SingleVertexWith2Committer(
+ boolean commit1Succeed, boolean commit2Succeed) throws IOException {
+ return createDAGPlan_SingleVertexWith2Committer(commit1Succeed, commit2Succeed, false);
+ }
+
+ // used for route event error in VM
+ private DAGPlan createDAGPlan_SingleVertexWith2Committer
+ (boolean commit1Succeed, boolean commit2Succeed, boolean customVM) throws IOException {
+ LOG.info("Setting up group dag plan");
+ int dummyTaskCount = 1;
+ Resource dummyTaskResource = Resource.newInstance(1, 1);
+ org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create(
+ "vertex1", ProcessorDescriptor.create("Processor"), dummyTaskCount,
+ dummyTaskResource);
+ if (customVM) {
+ v1.setVertexManagerPlugin(
+ VertexManagerPluginDescriptor.create(
+ FailOnVMEventReceivedlVertexManager.class.getName()));
+ }
+ OutputCommitterDescriptor ocd1 = OutputCommitterDescriptor.create(
+ CountingOutputCommitter.class.getName()).setUserPayload(
+ UserPayload.create(ByteBuffer
+ .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig(
+ !commit1Succeed, true).toUserPayload())));
+ OutputCommitterDescriptor ocd2 = OutputCommitterDescriptor.create(
+ CountingOutputCommitter.class.getName()).setUserPayload(
+ UserPayload.create(ByteBuffer
+ .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig(
+ !commit2Succeed, true).toUserPayload())));
+
+ DAG dag = DAG.create("testDag");
+ dag.addVertex(v1);
+ OutputDescriptor outDesc = OutputDescriptor.create("output.class");
+ v1.addDataSink("v1Out_1", DataSinkDescriptor.create(outDesc, ocd1, null));
+ v1.addDataSink("v1Out_2", DataSinkDescriptor.create(outDesc, ocd2, null));
+ return dag.createDag(conf, null, null, null, true);
+ }
+
+ private void initDAG(DAGImpl dag) {
+ dag.handle(new DAGEvent(dag.getID(), DAGEventType.DAG_INIT));
+ Assert.assertEquals(DAGState.INITED, dag.getState());
+ }
+
+ @SuppressWarnings("unchecked")
+ private void startDAG(DAGImpl impl) {
+ dispatcher.getEventHandler().handle(
+ new DAGEventStartDag(impl.getID(), null));
+ dispatcher.await();
+ Assert.assertEquals(DAGState.RUNNING, impl.getState());
+ }
+
+ @Test(timeout = 5000)
+ public void testVertexSucceedWithoutCommit() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ true);
+ setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertNull(v1.getTerminationCause());
+ Assert.assertTrue(v1.commitFutures.isEmpty());
+ CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_1");
+ CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_2");
+ Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
+ Assert.assertEquals(0, v1OutputCommitter_1.commitCounter);
+ Assert.assertEquals(0, v1OutputCommitter_1.abortCounter);
+
+ Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
+ Assert.assertEquals(0, v1OutputCommitter_2.commitCounter);
+ Assert.assertEquals(0, v1OutputCommitter_2.abortCounter);
+ }
+
+ @Test(timeout = 5000)
+ public void testVertexCommit_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.COMMITTING, v1.getState());
+ CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_1");
+ v1OutputCommitter_1.unblockCommit();
+ waitForCommitCompleted(v1, "v1Out_1");
+ // still in COMMITTING due to another pending commit
+ Assert.assertEquals(VertexState.COMMITTING, v1.getState());
+
+ CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_2");
+ v1OutputCommitter_2.unblockCommit();
+ waitUntil(v1, VertexState.SUCCEEDED);
+ Assert.assertNull(v1.getTerminationCause());
+ Assert.assertTrue(v1.commitFutures.isEmpty());
+
+ Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.commitCounter);
+ Assert.assertEquals(0, v1OutputCommitter_1.abortCounter);
+
+ Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.commitCounter);
+ Assert.assertEquals(0, v1OutputCommitter_2.abortCounter);
+ }
+
+ // the first commit fail which cause the second commit abort
+ @Test(timeout = 5000)
+ public void testVertexCommitFail1_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan_SingleVertexWith2Committer(false, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.COMMITTING, v1.getState());
+ CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_1");
+ v1OutputCommitter_1.unblockCommit();
+ waitUntil(v1, VertexState.FAILED);
+
+ Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE,
+ v1.getTerminationCause());
+ Assert.assertTrue(v1.commitFutures.isEmpty());
+ CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_2");
+ Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.commitCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
+
+ Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
+ // can't verify the commitCounter because v1OutputCommitter_2 may not be started
+ Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
+ }
+
+ // the first commit succeed while the second fails
+ @Test(timeout = 5000)
+ public void testVertexCommitFail2_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan_SingleVertexWith2Committer(true, false));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.COMMITTING, v1.getState());
+ CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_1");
+ v1OutputCommitter_1.unblockCommit();
+ waitForCommitCompleted(v1, "v1Out_1");
+ Assert.assertEquals(VertexState.COMMITTING, v1.getState());
+
+ CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_2");
+ v1OutputCommitter_2.unblockCommit();
+ waitUntil(v1, VertexState.FAILED);
+
+ Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE,
+ v1.getTerminationCause());
+ Assert.assertTrue(v1.commitFutures.isEmpty());
+ Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.commitCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
+
+ Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.commitCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
+ }
+
+ @Test(timeout = 5000)
+ public void testVertexKilledWhileCommitting() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.COMMITTING, v1.getState());
+ // kill dag which will trigger the vertex killed event
+ dag.handle(new DAGEvent(dag.getID(), DAGEventType.DAG_KILL));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.KILLED, v1.getState());
+ Assert.assertTrue(v1.commitFutures.isEmpty());
+ Assert.assertEquals(VertexTerminationCause.DAG_KILL,
+ v1.getTerminationCause());
+ Assert.assertEquals(DAGState.KILLED, dag.getState());
+ Assert
+ .assertEquals(DAGTerminationCause.DAG_KILL, dag.getTerminationCause());
+
+ CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_1");
+ CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_2");
+ Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
+
+ Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
+ }
+
+ @Test(timeout = 5000)
+ public void testVertexRescheduleWhileCommitting() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.COMMITTING, v1.getState());
+ // reschedule task
+ v1.handle(new VertexEventTaskReschedule(TezTaskID.getInstance(
+ v1.getVertexId(), 2)));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.FAILED, v1.getState());
+ Assert.assertEquals(VertexTerminationCause.VERTEX_RERUN_IN_COMMITTING,
+ v1.getTerminationCause());
+ Assert.assertTrue(v1.commitFutures.isEmpty());
+ Assert.assertEquals(DAGState.FAILED, dag.getState());
+ Assert.assertEquals(DAGTerminationCause.VERTEX_FAILURE,
+ dag.getTerminationCause());
+
+ CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_1");
+ CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_2");
+ Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
+
+ Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
+ }
+
+ @Test(timeout = 5000)
+ public void testVertexRouteEventErrorWhileCommitting() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.COMMITTING, v1.getState());
+ // reschedule task
+ VertexManagerEvent vmEvent = VertexManagerEvent.create("vertex1", ByteBuffer.wrap(new byte[0]));
+ TezEvent tezEvent = new TezEvent(vmEvent,
+ new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1",
+ null, null));
+ v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
+ waitUntil(dag, DAGState.FAILED);
+
+ Assert.assertEquals(VertexState.FAILED, v1.getState());
+ Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE,
+ v1.getTerminationCause());
+ Assert.assertTrue(v1.commitFutures.isEmpty());
+ Assert.assertEquals(DAGState.FAILED, dag.getState());
+ Assert.assertEquals(DAGTerminationCause.VERTEX_FAILURE,
+ dag.getTerminationCause());
+
+ CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_1");
+ CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_2");
+ Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
+
+ Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
+ }
+
+ @Test(timeout = 5000)
+ public void testVertexInternalErrorWhileCommiting() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.COMMITTING, v1.getState());
+ // internal error
+ v1.handle(new VertexEvent(v1.getVertexId(),
+ VertexEventType.V_INTERNAL_ERROR));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.ERROR, v1.getState());
+ Assert.assertEquals(VertexTerminationCause.INTERNAL_ERROR,
+ v1.getTerminationCause());
+ Assert.assertEquals(DAGState.ERROR, dag.getState());
+ Assert.assertEquals(DAGTerminationCause.INTERNAL_ERROR,
+ dag.getTerminationCause());
+
+ CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_1");
+ CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1
+ .getOutputCommitter("v1Out_2");
+ Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ // TODO abort it when internal error happens TEZ-2250
+ // Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
+
+ Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
+ Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ // TODO abort it when internal error happens TEZ-2250
+ // Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
+ }
+
+ @Test(timeout = 5000)
+ public void testDAGCommitSucceeded_OnDAGSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ true);
+ setupDAG(createDAGPlan(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ waitUntil(dag, DAGState.COMMITTING);
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ v12OutputCommitter.unblockCommit();
+ // still in COMMITTING due to another pending commit
+ waitUntil(dag, DAGState.COMMITTING);
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ v3OutputCommitter.unblockCommit();
+ waitUntil(dag, DAGState.SUCCEEDED);
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+ Assert.assertNull(dag.getTerminationCause());
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(0, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v3OutputCommitter.commitCounter);
+ Assert.assertEquals(0, v3OutputCommitter.abortCounter);
+ }
+
+ // first commit(v12Out) succeed and then the second commit(v3Out) fail
+ @Test(timeout = 5000)
+ public void testDAGCommitFail1_OnDAGSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ true);
+ setupDAG(createDAGPlan(true, false));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ waitUntil(dag, DAGState.COMMITTING);
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ v12OutputCommitter.unblockCommit();
+ waitForCommitCompleted(dag, new OutputKey("v12Out", "uv12", true));
+ // still in COMMITTING due to another pending commit
+ Assert.assertEquals(DAGState.COMMITTING, dag.getState());
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ v3OutputCommitter.unblockCommit();
+ waitUntil(dag, DAGState.FAILED);
+
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v3.getState());
+
+ Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE,
+ dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v3OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ // the first commit(v12Out) fail
+ @Test(timeout = 5000)
+ public void testDAGCommitFail2_OnDAGSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ true);
+ setupDAG(createDAGPlan(false, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ waitUntil(dag, DAGState.COMMITTING);
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ v12OutputCommitter.unblockCommit();
+ waitUntil(dag, DAGState.FAILED);
+
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v3.getState());
+
+ Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE,
+ dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ // commit of v3Out complete first then commit of v12Out complete
+ @Test(timeout = 5000)
+ public void testDAGCommitSucceeded1_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.COMMITTING, v3.getState());
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ v3OutputCommitter.unblockCommit();
+ waitUntil(v3, VertexState.SUCCEEDED);
+ // dag go to COMMITTING due to the pending vertex group commit of v1,v2
+ waitUntil(dag, DAGState.COMMITTING);
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ v12OutputCommitter.unblockCommit();
+ waitUntil(dag, DAGState.SUCCEEDED);
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(0, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v3OutputCommitter.commitCounter);
+ Assert.assertEquals(0, v3OutputCommitter.abortCounter);
+ }
+
+ // commit of v12Out complete first then commit of v3Out
+ @Test(timeout = 5000)
+ public void testDAGCommitSucceeded2_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.COMMITTING, v3.getState());
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ v12OutputCommitter.unblockCommit();
+ // ugly (wait for commit event sent out)
+ Thread.sleep(500);
+ dispatcher.await();
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ v3OutputCommitter.unblockCommit();
+ waitUntil(dag, DAGState.SUCCEEDED);
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(0, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v3OutputCommitter.commitCounter);
+ Assert.assertEquals(0, v3OutputCommitter.abortCounter);
+ }
+
+ // commit of vertex group(v1,v2) fail and commit of v3 is not completed
+ @Test(timeout = 5000)
+ public void testDAGCommitFail1_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan(false, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.COMMITTING, v3.getState());
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ v12OutputCommitter.unblockCommit();
+ waitUntil(dag, DAGState.FAILED);
+ // v3 is killed due to the commit failure of the vertex group (v1,v2)
+ Assert.assertEquals(VertexState.KILLED, v3.getState());
+ Assert.assertEquals(VertexTerminationCause.OTHER_VERTEX_FAILURE,
+ v3.getTerminationCause());
+ Assert.assertTrue(v3.commitFutures.isEmpty());
+ Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE,
+ dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ // commit of vertex v3 fail and commit of vertex group (v1,v2) is not completed
+ @Test(timeout = 5000)
+ public void testDAGCommitFail2_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan(true, false));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.COMMITTING, v3.getState());
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ v3OutputCommitter.unblockCommit();
+ waitUntil(dag, DAGState.FAILED);
+
+ Assert.assertEquals(VertexState.FAILED, v3.getState());
+ Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE,
+ v3.getTerminationCause());
+ Assert.assertTrue(v3.commitFutures.isEmpty());
+ Assert.assertEquals(DAGTerminationCause.VERTEX_FAILURE,
+ dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v3OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ // vertex group (v1,v2) succeeded first and then commit of vertex v3 fail
+ @Test (timeout = 5000)
+ public void testDAGCommitFail3_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan(true, false));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.COMMITTING, v3.getState());
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ v12OutputCommitter.unblockCommit();
+
+ waitForCommitCompleted(dag, new OutputKey("v12Out", "uv12", true));
+
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ v3OutputCommitter.unblockCommit();
+ waitUntil(dag, DAGState.FAILED);
+
+ Assert.assertEquals(VertexState.FAILED, v3.getState());
+ Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE,
+ v3.getTerminationCause());
+ Assert.assertTrue(v3.commitFutures.isEmpty());
+ Assert.assertEquals(DAGTerminationCause.VERTEX_FAILURE,
+ dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v3OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ // commit of vertex v3 succeeded first and then commit of vertex group(v1,v2) fail
+ @Test(timeout = 5000)
+ public void testDAGCommitFail4_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan(false, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.COMMITTING, v3.getState());
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ v3OutputCommitter.unblockCommit();
+ waitForCommitCompleted(dag, new OutputKey("v3Out", "vertex3", true));
+ waitUntil(v3, VertexState.SUCCEEDED);
+ Assert.assertTrue(v3.commitFutures.isEmpty());
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ v12OutputCommitter.unblockCommit();
+ waitUntil(dag, DAGState.FAILED);
+
+ Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE,
+ dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v3OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ @Test (timeout = 5000)
+ public void testDAGInternalErrorWhileCommiting() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ true);
+ setupDAG(createDAGPlan(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ waitUntil(dag, DAGState.COMMITTING);
+ dag.handle(new DAGEvent(dag.getID(), DAGEventType.INTERNAL_ERROR));
+ waitUntil(dag, DAGState.ERROR);
+
+ Assert.assertEquals(DAGTerminationCause.INTERNAL_ERROR, dag.getTerminationCause());
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ // TODO abort it when internal error happens TEZ-2250
+ // Assert.assertEquals(0, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ // TODO abort it when internal error happens TEZ-2250
+ // Assert.assertEquals(0, v3OutputCommitter.abortCounter);
+ }
+
+ // Kill dag while it is in COMMITTING in the case of
+ // TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS is true
+ @Test(timeout = 5000)
+ public void testDAGKilledWhileCommitting1_OnDAGSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ true);
+ setupDAG(createDAGPlan(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ waitUntil(dag, DAGState.COMMITTING);
+ dag.handle(new DAGEvent(dag.getID(), DAGEventType.DAG_KILL));
+ waitUntil(dag, DAGState.KILLED);
+
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v3.getState());
+ Assert
+ .assertEquals(DAGTerminationCause.DAG_KILL, dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ // Kill dag while it is in COMMITTING in the case of
+ // TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS is false
+ @Test(timeout = 5000)
+ public void testDAGKilledWhileCommitting1_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.COMMITTING, v3.getState());
+ // dag is still in RUNNING because v3 has not completed
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ v3OutputCommitter.unblockCommit();
+ // dag go to COMMITTING due to the pending commit of v12Out
+ waitUntil(dag, DAGState.COMMITTING);
+ dag.handle(new DAGEvent(dag.getID(), DAGEventType.DAG_KILL));
+ waitUntil(dag, DAGState.KILLED);
+
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v3.getState());
+ Assert.assertEquals(DAGState.KILLED, dag.getState());
+ Assert
+ .assertEquals(DAGTerminationCause.DAG_KILL, dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v3OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ // DAG killed while dag is still in RUNNING and vertex is in COMMITTING
+ @Test(timeout = 5000)
+ public void testDAGKilledWhileRunning_OnVertexSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.COMMITTING, v3.getState());
+ // dag is still in RUNNING because v3 has not completed
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+ dag.handle(new DAGEvent(dag.getID(), DAGEventType.DAG_KILL));
+ waitUntil(dag, DAGState.KILLED);
+
+ Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
+ Assert.assertEquals(VertexState.KILLED, v3.getState());
+ Assert.assertEquals(VertexTerminationCause.DAG_KILL, v3.getTerminationCause());
+ Assert.assertTrue(v3.commitFutures.isEmpty());
+ Assert.assertEquals(DAGState.KILLED, dag.getState());
+ Assert
+ .assertEquals(DAGTerminationCause.DAG_KILL, dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ @Test(timeout = 5000)
+ public void testDAGCommitVertexRerunWhileCommitting_OnDAGSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ true);
+ setupDAG(createDAGPlan(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ waitUntil(dag, DAGState.COMMITTING);
+ dag.handle(new DAGEventVertexReRunning(v1.getVertexId()));
+ waitUntil(dag, DAGState.FAILED);
+
+ Assert.assertEquals(DAGTerminationCause.VERTEX_RERUN_IN_COMMITTING, dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ @Test(timeout = 5000)
+ public void testDAGCommitInternalErrorWhileCommiting_OnDAGSuccess() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ true);
+ setupDAG(createDAGPlan(true, true));
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ waitUntil(dag, DAGState.COMMITTING);
+ dag.handle(new DAGEvent(dag.getID(), DAGEventType.INTERNAL_ERROR));
+ waitUntil(dag, DAGState.ERROR);
+
+ Assert.assertEquals(DAGTerminationCause.INTERNAL_ERROR, dag.getTerminationCause());
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ @Test(timeout = 5000)
+ public void testVertexGroupCommitFinishedEventFail() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ false);
+ setupDAG(createDAGPlan(true, true));
+ MockHistoryEventHandler mockHistoryEventHandler = new MockHistoryEventHandler(appContext);
+ doReturn(mockHistoryEventHandler).when(appContext).getHistoryHandler();
+ mockHistoryEventHandler.failVertexGroupCommitFinishedEvent = true;
+
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ v12OutputCommitter.unblockCommit();
+ waitUntil(dag, DAGState.FAILED);
+
+ Assert.assertEquals(DAGState.FAILED, dag.getState());
+ Assert.assertEquals(DAGTerminationCause.RECOVERY_FAILURE,
+ dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+ Assert.assertEquals(VertexState.KILLED, v3.getState());
+ Assert.assertEquals(VertexTerminationCause.OTHER_VERTEX_FAILURE, v3.getTerminationCause());
+ Assert.assertTrue(v3.commitFutures.isEmpty());
+
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ Assert.assertEquals(1, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ // commit may not have started, so can't verify commitCounter
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ @Test(timeout = 5000)
+ public void testDAGCommitStartedEventFail() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ true);
+ setupDAG(createDAGPlan(true, true));
+ MockHistoryEventHandler mockHistoryEventHandler = new MockHistoryEventHandler(appContext);
+ doReturn(mockHistoryEventHandler).when(appContext).getHistoryHandler();
+ mockHistoryEventHandler.failDAGCommitStartedEvent = true;
+
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ waitUntil(dag, DAGState.FAILED);
+ Assert.assertEquals(DAGTerminationCause.RECOVERY_FAILURE, dag.getTerminationCause());
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ // commit has not started
+ Assert.assertEquals(0, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ // commit has not started
+ Assert.assertEquals(0, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+ }
+
+ // test commit will be canceled no matter it is started or still in the threadpool
+ // ControlledThreadPoolExecutor is used for to not schedule the commits
+ @Test(timeout = 5000)
+ public void testCommitCanceled() throws Exception {
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ true);
+ setupDAG(createDAGPlan(true, true));
+ // create customized ThreadPoolExecutor to wait before schedule new task
+ rawExecutor = new ControlledThreadPoolExecutor(1);
+ execService = MoreExecutors.listeningDecorator(rawExecutor);
+ doReturn(execService).when(appContext).getExecService();
+
+ initDAG(dag);
+ startDAG(dag);
+ VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
+ VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
+ // need to make vertices to go to SUCCEEDED
+ v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(),
+ TaskState.SUCCEEDED));
+ waitUntil(dag, DAGState.COMMITTING);
+ // mean the commits have been submitted to ThreadPool
+ Assert.assertEquals(2, dag.commitFutures.size());
+
+ dag.handle(new DAGEvent(dag.getID(), DAGEventType.DAG_KILL));
+ waitUntil(dag, DAGState.KILLED);
+
+ Assert.assertEquals(DAGTerminationCause.DAG_KILL, dag.getTerminationCause());
+ // mean the commits have been canceled
+ Assert.assertTrue(dag.commitFutures.isEmpty());
+
+ CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1
+ .getOutputCommitter("v12Out");
+ CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3
+ .getOutputCommitter("v3Out");
+ Assert.assertEquals(1, v12OutputCommitter.initCounter);
+ Assert.assertEquals(1, v12OutputCommitter.setupCounter);
+ // commit is not started because ControlledThreadPoolExecutor wait before schedule tasks
+ Assert.assertEquals(0, v12OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v12OutputCommitter.abortCounter);
+
+ Assert.assertEquals(1, v3OutputCommitter.initCounter);
+ Assert.assertEquals(1, v3OutputCommitter.setupCounter);
+ // commit is not started because ControlledThreadPoolExecutor wait before schedule tasks
+ Assert.assertEquals(0, v3OutputCommitter.commitCounter);
+ Assert.assertEquals(1, v3OutputCommitter.abortCounter);
+
+ }
+
+ public static class FailOnVMEventReceivedlVertexManager extends ImmediateStartVertexManager {
+
+ public FailOnVMEventReceivedlVertexManager(VertexManagerPluginContext context) {
+ super(context);
+ }
+
+ @Override
+ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) {
+ super.onVertexManagerEventReceived(vmEvent);
+ throw new RuntimeException("fail vm");
+ }
+
+ }
+
+ private static class MockHistoryEventHandler extends HistoryEventHandler {
+
+ public boolean failVertexGroupCommitFinishedEvent = false;
+ public boolean failDAGCommitStartedEvent = false;
+
+ public MockHistoryEventHandler(AppContext context) {
+ super(context);
+ }
+
+ @Override
+ public void handleCriticalEvent(DAGHistoryEvent event) throws IOException {
+ if (event.getHistoryEvent().getEventType() == HistoryEventType.VERTEX_GROUP_COMMIT_FINISHED
+ && failVertexGroupCommitFinishedEvent) {
+ throw new IOException("fail VertexGroupCommitFinishedEvent");
+ }
+ if (event.getHistoryEvent().getEventType() == HistoryEventType.DAG_COMMIT_STARTED
+ && failDAGCommitStartedEvent) {
+ throw new IOException("fail DAGCommitStartedEvent");
+ }
+ }
+ }
+
+ private static class ControlledThreadPoolExecutor extends ThreadPoolExecutor {
+
+ public ControlledThreadPoolExecutor(int poolSize) {
+ this(poolSize, poolSize, 0L, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>());
+ }
+
+ public ControlledThreadPoolExecutor(int corePoolSize, int maximumPoolSize,
+ long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue) {
+ super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue);
+ }
+
+ public boolean startFlag = false;
+
+ @Override
+ protected void beforeExecute(Thread t, Runnable r) {
+ while(!startFlag) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ super.beforeExecute(t, r);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 98c8492..7e944ef 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -1364,7 +1364,7 @@ public class TestDAGImpl {
}
}
- // error on vertex -> dag error -> successful vertex output not aborted
+ // error on vertex -> dag error
Vertex errorVertex = mrrDag.getVertex("vertex3");
dispatcher.getEventHandler().handle(new VertexEvent(
errorVertex.getVertexId(), VertexEventType.V_INTERNAL_ERROR));
@@ -1383,7 +1383,8 @@ public class TestDAGImpl {
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
} else {
- Assert.assertEquals(0, committer.abortCounter);
+ // abort operation should take no side effort on the successful commit
+ Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(1, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 5ad320e..891da23 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -3341,8 +3341,7 @@ public class TestVertexImpl {
Assert.assertEquals(1, committer.commitCounter);
- // FIXME need to verify whether abort needs to be called if commit fails
- Assert.assertEquals(0, committer.abortCounter);
+ Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
@@ -3386,8 +3385,7 @@ public class TestVertexImpl {
Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE, v.getTerminationCause());
Assert.assertEquals(1, committer.commitCounter);
- // FIXME need to verify whether abort needs to be called if commit fails
- Assert.assertEquals(0, committer.abortCounter);
+ Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
index 185f4a4..d1bc46c 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
@@ -82,6 +82,8 @@ public class ExampleDriver {
"Filters lines by the specified word using broadcast edge");
pgd.addClass("filterLinesByWordOneToOne", FilterLinesByWordOneToOne.class,
"Filters lines by the specified word using OneToOne edge");
+ pgd.addClass("multiplecommitsExample", MultipleCommitsExample.class,
+ "Job with multiple commits in both vertex group and vertex");
exitCode = pgd.run(argv);
}
catch(Throwable e){
[34/50] [abbrv] tez git commit: TEZ-2090. Add tests for jobs running
in external services. (sseth)
Posted by ss...@apache.org.
TEZ-2090. Add tests for jobs running in external services. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/78d4ebc1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/78d4ebc1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/78d4ebc1
Branch: refs/heads/TEZ-2003
Commit: 78d4ebc12ad4ae3c0ea7c6e9af81d75fa166390e
Parents: c3c2285
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Feb 13 17:24:05 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:44:32 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
pom.xml | 6 +
.../apache/tez/dag/api/TezConfiguration.java | 2 +
.../apache/tez/dag/api/TaskCommunicator.java | 1 +
.../tez/dag/api/TaskCommunicatorContext.java | 3 +
.../tez/dag/app/TezTaskCommunicatorImpl.java | 42 +-
.../dag/app/rm/TaskSchedulerEventHandler.java | 2 +-
tez-ext-service-tests/pom.xml | 161 ++++
.../tez/dag/app/TezTestServiceCommunicator.java | 152 ++++
.../TezTestServiceContainerLauncher.java | 144 ++++
.../TezTestServiceNoOpContainerLauncher.java | 66 ++
.../rm/TezTestServiceTaskSchedulerService.java | 347 ++++++++
.../TezTestServiceTaskCommunicatorImpl.java | 182 ++++
.../org/apache/tez/service/ContainerRunner.java | 27 +
.../tez/service/MiniTezTestServiceCluster.java | 163 ++++
.../service/TezTestServiceConfConstants.java | 41 +
.../TezTestServiceProtocolBlockingPB.java | 22 +
.../tez/service/impl/ContainerRunnerImpl.java | 512 +++++++++++
.../apache/tez/service/impl/TezTestService.java | 126 +++
.../impl/TezTestServiceProtocolClientImpl.java | 82 ++
.../impl/TezTestServiceProtocolServerImpl.java | 133 +++
.../tez/shufflehandler/FadvisedChunkedFile.java | 78 ++
.../tez/shufflehandler/FadvisedFileRegion.java | 160 ++++
.../apache/tez/shufflehandler/IndexCache.java | 199 +++++
.../tez/shufflehandler/ShuffleHandler.java | 840 +++++++++++++++++++
.../tez/tests/TestExternalTezServices.java | 183 ++++
.../org/apache/tez/util/ProtoConverters.java | 172 ++++
.../src/test/proto/TezDaemonProtocol.proto | 84 ++
.../src/test/resources/log4j.properties | 19 +
.../org/apache/tez/runtime/task/TezChild.java | 2 +-
.../apache/tez/runtime/task/TezTaskRunner.java | 2 +-
31 files changed, 3943 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index d7e4be5..975ce65 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -1,5 +1,6 @@
ALL CHANGES:
TEZ-2019. Temporarily allow the scheduler and launcher to be specified via configuration.
TEZ-2006. Task communication plane needs to be pluggable.
+ TEZ-2090. Add tests for jobs running in external services.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1e29c4d..50834d3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -169,6 +169,11 @@
<type>test-jar</type>
</dependency>
<dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-ext-service-tests</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.5</version>
@@ -631,6 +636,7 @@
<module>tez-dag</module>
<module>tez-ui</module>
<module>tez-plugins</module>
+ <module>tez-ext-service-tests</module>
<module>tez-dist</module>
<module>docs</module>
</modules>
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 3509509..f71d9dd 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1140,6 +1140,8 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_AM_CONTAINER_LAUNCHER_CLASS = TEZ_AM_PREFIX + "container-launcher.class";
@ConfigurationScope(Scope.VERTEX)
public static final String TEZ_AM_TASK_SCHEDULER_CLASS = TEZ_AM_PREFIX + "task-scheduler.class";
+ @ConfigurationScope(Scope.VERTEX)
+ public static final String TEZ_AM_TASK_COMMUNICATOR_CLASS = TEZ_AM_PREFIX + "task-communicator.class";
// TODO only validate property here, value can also be validated if necessary
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
index 97f9c16..c9f85e0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
@@ -14,6 +14,7 @@
package org.apache.tez.dag.api;
+import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
index 9b2d889..41675fe 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
@@ -44,5 +44,8 @@ public interface TaskCommunicatorContext {
// TODO TEZ-2003 Move to vertex, taskIndex, version
void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId);
+ // TODO TEZ-2003 Add an API to register task failure - for example, a communication failure.
+ // This will have to take into consideration the TA_FAILED event
+
// TODO Eventually Add methods to report availability stats to the scheduler.
}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
index 5652937..258c927 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
@@ -74,16 +74,22 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
new ConcurrentHashMap<TaskAttempt, ContainerId>();
private final TezTaskUmbilicalProtocol taskUmbilical;
+ private final String tokenIdentifier;
+ private final Token<JobTokenIdentifier> sessionToken;
private InetSocketAddress address;
private Server server;
- private static final class ContainerInfo {
+ public static final class ContainerInfo {
- ContainerInfo(ContainerId containerId) {
+ ContainerInfo(ContainerId containerId, String host, int port) {
this.containerId = containerId;
+ this.host = host;
+ this.port = port;
}
- ContainerId containerId;
+ final ContainerId containerId;
+ public final String host;
+ public final int port;
TezHeartbeatResponse lastResponse = null;
TaskSpec taskSpec = null;
long lastRequestId = 0;
@@ -110,6 +116,8 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
super(TezTaskCommunicatorImpl.class.getName());
this.taskCommunicatorContext = taskCommunicatorContext;
this.taskUmbilical = new TezTaskUmbilicalProtocolImpl();
+ this.tokenIdentifier = this.taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString();
+ this.sessionToken = TokenCache.getSessionToken(taskCommunicatorContext.getCredentials());
}
@@ -130,9 +138,7 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
try {
JobTokenSecretManager jobTokenSecretManager =
new JobTokenSecretManager();
- Token<JobTokenIdentifier> sessionToken = TokenCache.getSessionToken(taskCommunicatorContext.getCredentials());
- jobTokenSecretManager.addTokenForJob(
- taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString(), sessionToken);
+ jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken);
server = new RPC.Builder(conf)
.setProtocol(TezTaskUmbilicalProtocol.class)
@@ -182,7 +188,7 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
@Override
public void registerRunningContainer(ContainerId containerId, String host, int port) {
- ContainerInfo oldInfo = registeredContainers.putIfAbsent(containerId, new ContainerInfo(containerId));
+ ContainerInfo oldInfo = registeredContainers.putIfAbsent(containerId, new ContainerInfo(containerId, host, port));
if (oldInfo != null) {
throw new TezUncheckedException("Multiple registrations for containerId: " + containerId);
}
@@ -230,9 +236,9 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
". Already registered to containerId: " + oldId);
}
}
-
}
+
@Override
public void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID) {
TaskAttempt taskAttempt = new TaskAttempt(taskAttemptID);
@@ -258,6 +264,18 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
return address;
}
+ protected String getTokenIdentifier() {
+ return tokenIdentifier;
+ }
+
+ protected Token<JobTokenIdentifier> getSessionToken() {
+ return sessionToken;
+ }
+
+ protected TaskCommunicatorContext getTaskCommunicatorContext() {
+ return taskCommunicatorContext;
+ }
+
public TezTaskUmbilicalProtocol getUmbilical() {
return this.taskUmbilical;
}
@@ -471,4 +489,12 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
return "TaskAttempt{" + "taskAttemptId=" + taskAttemptId + '}';
}
}
+
+ protected ContainerInfo getContainerInfo(ContainerId containerId) {
+ return registeredContainers.get(containerId);
+ }
+
+ protected ContainerId getContainerForAttempt(TezTaskAttemptID taskAttemptId) {
+ return attemptToContainerMap.get(new TaskAttempt(taskAttemptId));
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 62f82db..8c3ed87 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -350,7 +350,7 @@ public class TaskSchedulerEventHandler extends AbstractService
try {
Constructor<? extends TaskSchedulerService> ctor = taskSchedulerClazz
.getConstructor(TaskSchedulerAppCallback.class, AppContext.class, String.class,
- Integer.class, String.class, Configuration.class);
+ int.class, String.class, Configuration.class);
ctor.setAccessible(true);
TaskSchedulerService taskSchedulerService =
ctor.newInstance(this, appContext, host, port, trackingUrl, getConfig());
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml
new file mode 100644
index 0000000..37f68b1
--- /dev/null
+++ b/tez-ext-service-tests/pom.xml
@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>tez</artifactId>
+ <groupId>org.apache.tez</groupId>
+ <version>0.7.0-SNAPSHOT</version>
+ </parent>
+
+ <!-- TODO TEZ-2003 Merge this into the tez-tests module -->
+ <artifactId>tez-ext-service-tests</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-internals</artifactId>
+ </dependency>
+ <dependency>
+ <!-- Required for the ShuffleHandler -->
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-library</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-dag</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-tests</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <!--
+ Include all files in src/main/resources. By default, do not apply property
+ substitution (filtering=false), but do apply property substitution to
+ version-info.properties (filtering=true). This will substitute the
+ version information correctly, but prevent Maven from altering other files.
+ -->
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <excludes>
+ <exclude>tez-api-version-info.properties</exclude>
+ </excludes>
+ <filtering>false</filtering>
+ </resource>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>tez-api-version-info.properties</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
+ <imports>
+ <param>${basedir}/src/test/proto</param>
+ <param>${basedir}/../tez-api/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/test/proto</directory>
+ <includes>
+ <include>TezDaemonProtocol.proto</include>
+ </includes>
+ </source>
+ <output>${project.build.directory}/generated-test-sources/java</output>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java
new file mode 100644
index 0000000..ac50878
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app;
+
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.protobuf.Message;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.tez.service.TezTestServiceProtocolBlockingPB;
+import org.apache.tez.service.impl.TezTestServiceProtocolClientImpl;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto;
+
+public class TezTestServiceCommunicator extends AbstractService {
+
+ private final ConcurrentMap<String, TezTestServiceProtocolBlockingPB> hostProxies;
+ private final ListeningExecutorService executor;
+
+ // TODO Convert this into a singleton
+ public TezTestServiceCommunicator(int numThreads) {
+ super(TezTestServiceCommunicator.class.getSimpleName());
+ ExecutorService localExecutor = Executors.newFixedThreadPool(numThreads,
+ new ThreadFactoryBuilder().setNameFormat("TezTestServiceCommunicator #%2d").build());
+ this.hostProxies = new ConcurrentHashMap<String, TezTestServiceProtocolBlockingPB>();
+ executor = MoreExecutors.listeningDecorator(localExecutor);
+ }
+
+ @Override
+ public void serviceStop() {
+ executor.shutdownNow();
+ }
+
+
+ public void runContainer(RunContainerRequestProto request, String host, int port,
+ final ExecuteRequestCallback<RunContainerResponseProto> callback) {
+ ListenableFuture<RunContainerResponseProto> future = executor.submit(new RunContainerCallable(request, host, port));
+ Futures.addCallback(future, new FutureCallback<RunContainerResponseProto>() {
+ @Override
+ public void onSuccess(RunContainerResponseProto result) {
+ callback.setResponse(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ callback.indicateError(t);
+ }
+ });
+
+ }
+
+ public void submitWork(SubmitWorkRequestProto request, String host, int port,
+ final ExecuteRequestCallback<SubmitWorkResponseProto> callback) {
+ ListenableFuture<SubmitWorkResponseProto> future = executor.submit(new SubmitWorkCallable(request, host, port));
+ Futures.addCallback(future, new FutureCallback<SubmitWorkResponseProto>() {
+ @Override
+ public void onSuccess(SubmitWorkResponseProto result) {
+ callback.setResponse(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ callback.indicateError(t);
+ }
+ });
+
+ }
+
+
+ private class RunContainerCallable implements Callable<RunContainerResponseProto> {
+
+ final String hostname;
+ final int port;
+ final RunContainerRequestProto request;
+
+ private RunContainerCallable(RunContainerRequestProto request, String hostname, int port) {
+ this.hostname = hostname;
+ this.port = port;
+ this.request = request;
+ }
+
+ @Override
+ public RunContainerResponseProto call() throws Exception {
+ return getProxy(hostname, port).runContainer(null, request);
+ }
+ }
+
+ private class SubmitWorkCallable implements Callable<SubmitWorkResponseProto> {
+ final String hostname;
+ final int port;
+ final SubmitWorkRequestProto request;
+
+ private SubmitWorkCallable(SubmitWorkRequestProto request, String hostname, int port) {
+ this.hostname = hostname;
+ this.port = port;
+ this.request = request;
+ }
+
+ @Override
+ public SubmitWorkResponseProto call() throws Exception {
+ return getProxy(hostname, port).submitWork(null, request);
+ }
+ }
+
+ public interface ExecuteRequestCallback<T extends Message> {
+ void setResponse(T response);
+ void indicateError(Throwable t);
+ }
+
+ private TezTestServiceProtocolBlockingPB getProxy(String hostname, int port) {
+ String hostId = getHostIdentifier(hostname, port);
+
+ TezTestServiceProtocolBlockingPB proxy = hostProxies.get(hostId);
+ if (proxy == null) {
+ proxy = new TezTestServiceProtocolClientImpl(getConfig(), hostname, port);
+ TezTestServiceProtocolBlockingPB proxyOld = hostProxies.putIfAbsent(hostId, proxy);
+ if (proxyOld != null) {
+ // TODO Shutdown the new proxy.
+ proxy = proxyOld;
+ }
+ }
+ return proxy;
+ }
+
+ private String getHostIdentifier(String hostname, int port) {
+ return hostname + ":" + port;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
new file mode 100644
index 0000000..e83165b
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.launcher;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TezTestServiceCommunicator;
+import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
+import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
+import org.apache.tez.dag.app.rm.container.AMContainerEvent;
+import org.apache.tez.dag.app.rm.container.AMContainerEventLaunchFailed;
+import org.apache.tez.dag.app.rm.container.AMContainerEventLaunched;
+import org.apache.tez.dag.app.rm.container.AMContainerEventType;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
+import org.apache.tez.service.TezTestServiceConfConstants;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+
+public class TezTestServiceContainerLauncher extends AbstractService implements ContainerLauncher {
+
+ // TODO Support interruptability of tasks which haven't yet been launched.
+
+ // TODO May need multiple connections per target machine, depending upon how synchronization is handled in the RPC layer
+
+ static final Log LOG = LogFactory.getLog(TezTestServiceContainerLauncher.class);
+
+ private final AppContext context;
+ private final String tokenIdentifier;
+ private final TaskAttemptListener tal;
+ private final int servicePort;
+ private final TezTestServiceCommunicator communicator;
+ private final Clock clock;
+
+
+ // Configuration passed in here to set up final parameters
+ public TezTestServiceContainerLauncher(AppContext appContext, Configuration conf,
+ TaskAttemptListener tal) {
+ super(TezTestServiceContainerLauncher.class.getName());
+ this.clock = appContext.getClock();
+ int numThreads = conf.getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS,
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT);
+
+ this.servicePort = conf.getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1);
+ Preconditions.checkArgument(servicePort > 0,
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be set");
+ this.communicator = new TezTestServiceCommunicator(numThreads);
+ this.context = appContext;
+ this.tokenIdentifier = context.getApplicationID().toString();
+ this.tal = tal;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ communicator.init(conf);
+ }
+
+ @Override
+ public void serviceStart() {
+ communicator.start();
+ }
+
+ @Override
+ public void serviceStop() {
+ communicator.stop();
+ }
+
+ @Override
+ public void handle(NMCommunicatorEvent event) {
+ switch (event.getType()) {
+ case CONTAINER_LAUNCH_REQUEST:
+ final NMCommunicatorLaunchRequestEvent launchEvent = (NMCommunicatorLaunchRequestEvent) event;
+ RunContainerRequestProto runRequest = constructRunContainerRequest(launchEvent);
+ communicator.runContainer(runRequest, launchEvent.getNodeId().getHost(),
+ launchEvent.getNodeId().getPort(),
+ new TezTestServiceCommunicator.ExecuteRequestCallback<TezTestServiceProtocolProtos.RunContainerResponseProto>() {
+ @Override
+ public void setResponse(TezTestServiceProtocolProtos.RunContainerResponseProto response) {
+ LOG.info("Container: " + launchEvent.getContainerId() + " launch succeeded on host: " + launchEvent.getNodeId());
+ context.getEventHandler().handle(new AMContainerEventLaunched(launchEvent.getContainerId()));
+ ContainerLaunchedEvent lEvt = new ContainerLaunchedEvent(
+ launchEvent.getContainerId(), clock.getTime(), context.getApplicationAttemptId());
+ context.getHistoryHandler().handle(new DAGHistoryEvent(
+ null, lEvt));
+ }
+
+ @Override
+ public void indicateError(Throwable t) {
+ LOG.error("Failed to launch container: " + launchEvent.getContainer() + " on host: " + launchEvent.getNodeId(), t);
+ sendContainerLaunchFailedMsg(launchEvent.getContainerId(), t);
+ }
+ });
+ break;
+ case CONTAINER_STOP_REQUEST:
+ LOG.info("DEBUG: Ignoring STOP_REQUEST for event: " + event);
+ // that the container is actually done (normally received from RM)
+ // TODO Sending this out for an un-launched container is invalid
+ context.getEventHandler().handle(new AMContainerEvent(event.getContainerId(),
+ AMContainerEventType.C_NM_STOP_SENT));
+ break;
+ }
+ }
+
+ private RunContainerRequestProto constructRunContainerRequest(NMCommunicatorLaunchRequestEvent event) {
+ RunContainerRequestProto.Builder builder = RunContainerRequestProto.newBuilder();
+ builder.setAmHost(tal.getAddress().getHostName()).setAmPort(tal.getAddress().getPort());
+ builder.setAppAttemptNumber(event.getContainer().getId().getApplicationAttemptId().getAttemptId());
+ builder.setApplicationIdString(
+ event.getContainer().getId().getApplicationAttemptId().getApplicationId().toString());
+ builder.setTokenIdentifier(tokenIdentifier);
+ builder.setContainerIdString(event.getContainer().getId().toString());
+ builder.setCredentialsBinary(
+ ByteString.copyFrom(event.getContainerLaunchContext().getTokens()));
+ // TODO Avoid reading this from the environment
+ builder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
+ return builder.build();
+ }
+
+ @SuppressWarnings("unchecked")
+ void sendContainerLaunchFailedMsg(ContainerId containerId, Throwable t) {
+ context.getEventHandler().handle(new AMContainerEventLaunchFailed(containerId, t == null ? "" : t.getMessage()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java
new file mode 100644
index 0000000..8c8e486
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.launcher;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
+import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
+import org.apache.tez.dag.app.rm.container.AMContainerEvent;
+import org.apache.tez.dag.app.rm.container.AMContainerEventLaunched;
+import org.apache.tez.dag.app.rm.container.AMContainerEventType;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
+
+public class TezTestServiceNoOpContainerLauncher extends AbstractService implements ContainerLauncher {
+
+ static final Log LOG = LogFactory.getLog(TezTestServiceNoOpContainerLauncher.class);
+
+ private final AppContext context;
+ private final Clock clock;
+
+ public TezTestServiceNoOpContainerLauncher(AppContext appContext, Configuration conf,
+ TaskAttemptListener tal) {
+ super(TezTestServiceNoOpContainerLauncher.class.getName());
+ this.context = appContext;
+ this.clock = appContext.getClock();
+ }
+
+ @Override
+ public void handle(NMCommunicatorEvent event) {
+ switch(event.getType()) {
+ case CONTAINER_LAUNCH_REQUEST:
+ final NMCommunicatorLaunchRequestEvent launchEvent = (NMCommunicatorLaunchRequestEvent) event;
+ LOG.info("No-op launch for container: " + launchEvent.getContainerId() + " succeeded on host: " + launchEvent.getNodeId());
+ context.getEventHandler().handle(new AMContainerEventLaunched(launchEvent.getContainerId()));
+ ContainerLaunchedEvent lEvt = new ContainerLaunchedEvent(
+ launchEvent.getContainerId(), clock.getTime(), context.getApplicationAttemptId());
+ context.getHistoryHandler().handle(new DAGHistoryEvent(
+ null, lEvt));
+ break;
+ case CONTAINER_STOP_REQUEST:
+ LOG.info("DEBUG: Ignoring STOP_REQUEST for event: " + event);
+ context.getEventHandler().handle(new AMContainerEvent(event.getContainerId(),
+ AMContainerEventType.C_NM_STOP_SENT));
+ break;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
new file mode 100644
index 0000000..e3c18bf
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
@@ -0,0 +1,347 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Ints;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.service.TezTestServiceConfConstants;
+
+
+// TODO Registration with RM - so that the AM is considered dead and restarted in the expiry interval - 10 minutes.
+
+public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
+
+ private static final Log LOG = LogFactory.getLog(TezTestServiceTaskSchedulerService.class);
+
+ private final ExecutorService appCallbackExecutor;
+ private final TaskSchedulerAppCallback appClientDelegate;
+ private final AppContext appContext;
+ private final List<String> serviceHosts;
+ private final ContainerFactory containerFactory;
+ private final Random random = new Random();
+ // Currently all services must be running on the same port.
+ private final int containerPort;
+
+ private final String clientHostname;
+ private final int clientPort;
+ private final String trackingUrl;
+ private final AtomicBoolean isStopped = new AtomicBoolean(false);
+ private final ConcurrentMap<Object, ContainerId> runningTasks =
+ new ConcurrentHashMap<Object, ContainerId>();
+
+ private final AMRMClientAsync<AMRMClient.ContainerRequest> amRmClient;
+
+ // Per instance
+ private final int memoryPerInstance;
+ private final int coresPerInstance;
+ private final int executorsPerInstance;
+
+ // Per Executor Thread
+ private final Resource resourcePerContainer;
+
+
+ public TezTestServiceTaskSchedulerService(TaskSchedulerAppCallback appClient,
+ AppContext appContext,
+ String clientHostname, int clientPort,
+ String trackingUrl,
+ Configuration conf) {
+ // Accepting configuration here to allow setting up fields as final
+ super(TezTestServiceTaskSchedulerService.class.getName());
+ this.appCallbackExecutor = createAppCallbackExecutorService();
+ this.appClientDelegate = createAppCallbackDelegate(appClient);
+ this.appContext = appContext;
+ this.serviceHosts = new LinkedList<String>();
+ this.containerFactory = new ContainerFactory(appContext);
+
+ this.memoryPerInstance = conf
+ .getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB, -1);
+ Preconditions.checkArgument(memoryPerInstance > 0,
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB +
+ " must be configured");
+
+ this.executorsPerInstance = conf.getInt(
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE,
+ -1);
+ Preconditions.checkArgument(executorsPerInstance > 0,
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE +
+ " must be configured");
+
+ this.coresPerInstance = conf
+ .getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_VCPUS_PER_INSTANCE,
+ executorsPerInstance);
+
+ this.containerPort = conf.getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1);
+ Preconditions.checkArgument(executorsPerInstance > 0,
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be configured");
+
+ this.clientHostname = clientHostname;
+ this.clientPort = clientPort;
+ this.trackingUrl = trackingUrl;
+
+ int memoryPerContainer = (int) (memoryPerInstance / (float) executorsPerInstance);
+ int coresPerContainer = (int) (coresPerInstance / (float) executorsPerInstance);
+ this.resourcePerContainer = Resource.newInstance(memoryPerContainer, coresPerContainer);
+ this.amRmClient = TezAMRMClientAsync.createAMRMClientAsync(5000, new FakeAmRmCallbackHandler());
+
+ String[] hosts = conf.getTrimmedStrings(TezTestServiceConfConstants.TEZ_TEST_SERVICE_HOSTS);
+ if (hosts == null || hosts.length == 0) {
+ hosts = new String[]{"localhost"};
+ }
+ for (String host : hosts) {
+ serviceHosts.add(host);
+ }
+
+ LOG.info("Running with configuration: " +
+ "memoryPerInstance=" + memoryPerInstance +
+ ", vcoresPerInstance=" + coresPerInstance +
+ ", executorsPerInstance=" + executorsPerInstance +
+ ", resourcePerContainerInferred=" + resourcePerContainer +
+ ", hosts=" + serviceHosts.toString());
+
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ amRmClient.init(conf);
+ }
+
+ @Override
+ public void serviceStart() {
+ amRmClient.start();
+ RegisterApplicationMasterResponse response;
+ try {
+ amRmClient.registerApplicationMaster(clientHostname, clientPort, trackingUrl);
+ } catch (YarnException e) {
+ throw new TezUncheckedException(e);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+
+ @Override
+ public void serviceStop() {
+ if (!this.isStopped.getAndSet(true)) {
+
+ try {
+ TaskSchedulerAppCallback.AppFinalStatus status = appClientDelegate.getFinalAppStatus();
+ amRmClient.unregisterApplicationMaster(status.exitStatus, status.exitMessage,
+ status.postCompletionTrackingUrl);
+ } catch (YarnException e) {
+ throw new TezUncheckedException(e);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ appCallbackExecutor.shutdownNow();
+ }
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ // TODO This needs information about all running executors, and the amount of memory etc available across the cluster.
+ return Resource
+ .newInstance(Ints.checkedCast(serviceHosts.size() * memoryPerInstance),
+ serviceHosts.size() * coresPerInstance);
+ }
+
+ @Override
+ public int getClusterNodeCount() {
+ return serviceHosts.size();
+ }
+
+ @Override
+ public void resetMatchLocalityForAllHeldContainers() {
+ }
+
+ @Override
+ public Resource getTotalResources() {
+ return Resource
+ .newInstance(Ints.checkedCast(serviceHosts.size() * memoryPerInstance),
+ serviceHosts.size() * coresPerInstance);
+ }
+
+ @Override
+ public void blacklistNode(NodeId nodeId) {
+ LOG.info("DEBUG: BlacklistNode not supported");
+ }
+
+ @Override
+ public void unblacklistNode(NodeId nodeId) {
+ LOG.info("DEBUG: unBlacklistNode not supported");
+ }
+
+ @Override
+ public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks,
+ Priority priority, Object containerSignature, Object clientCookie) {
+ String host = selectHost(hosts);
+ Container container =
+ containerFactory.createContainer(resourcePerContainer, priority, host, containerPort);
+ runningTasks.put(task, container.getId());
+ appClientDelegate.taskAllocated(task, clientCookie, container);
+ }
+
+
+ @Override
+ public void allocateTask(Object task, Resource capability, ContainerId containerId,
+ Priority priority, Object containerSignature, Object clientCookie) {
+ String host = selectHost(null);
+ Container container =
+ containerFactory.createContainer(resourcePerContainer, priority, host, containerPort);
+ runningTasks.put(task, container.getId());
+ appClientDelegate.taskAllocated(task, clientCookie, container);
+ }
+
+ @Override
+ public boolean deallocateTask(Object task, boolean taskSucceeded) {
+ ContainerId containerId = runningTasks.remove(task);
+ if (containerId == null) {
+ LOG.error("Could not determine ContainerId for task: " + task +
+ " . Could have hit a race condition. Ignoring." +
+ " The query may hang since this \"unknown\" container is now taking up a slot permanently");
+ return false;
+ }
+ appClientDelegate.containerBeingReleased(containerId);
+ return true;
+ }
+
+ @Override
+ public Object deallocateContainer(ContainerId containerId) {
+ LOG.info("DEBUG: Ignoring deallocateContainer for containerId: " + containerId);
+ return null;
+ }
+
+ @Override
+ public void setShouldUnregister() {
+
+ }
+
+ @Override
+ public boolean hasUnregistered() {
+ // Nothing to do. No registration involved.
+ return true;
+ }
+
+ private ExecutorService createAppCallbackExecutorService() {
+ return Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
+ .setNameFormat("TaskSchedulerAppCaller #%d").setDaemon(true).build());
+ }
+
+ private TaskSchedulerAppCallback createAppCallbackDelegate(
+ TaskSchedulerAppCallback realAppClient) {
+ return new TaskSchedulerAppCallbackWrapper(realAppClient,
+ appCallbackExecutor);
+ }
+
+ private String selectHost(String[] requestedHosts) {
+ String host = null;
+ if (requestedHosts != null && requestedHosts.length > 0) {
+ Arrays.sort(requestedHosts);
+ host = requestedHosts[0];
+ LOG.info("Selected host: " + host + " from requested hosts: " + Arrays.toString(requestedHosts));
+ } else {
+ host = serviceHosts.get(random.nextInt(serviceHosts.size()));
+ LOG.info("Selected random host: " + host + " since the request contained no host information");
+ }
+ return host;
+ }
+
+ static class ContainerFactory {
+ final AppContext appContext;
+ AtomicInteger nextId;
+
+ public ContainerFactory(AppContext appContext) {
+ this.appContext = appContext;
+ this.nextId = new AtomicInteger(2);
+ }
+
+ public Container createContainer(Resource capability, Priority priority, String hostname, int port) {
+ ApplicationAttemptId appAttemptId = appContext.getApplicationAttemptId();
+ ContainerId containerId = ContainerId.newInstance(appAttemptId, nextId.getAndIncrement());
+ NodeId nodeId = NodeId.newInstance(hostname, port);
+ String nodeHttpAddress = "hostname:0";
+
+ Container container = Container.newInstance(containerId,
+ nodeId,
+ nodeHttpAddress,
+ capability,
+ priority,
+ null);
+
+ return container;
+ }
+ }
+
+ private static class FakeAmRmCallbackHandler implements AMRMClientAsync.CallbackHandler {
+
+ @Override
+ public void onContainersCompleted(List<ContainerStatus> statuses) {
+
+ }
+
+ @Override
+ public void onContainersAllocated(List<Container> containers) {
+
+ }
+
+ @Override
+ public void onShutdownRequest() {
+
+ }
+
+ @Override
+ public void onNodesUpdated(List<NodeReport> updatedNodes) {
+
+ }
+
+ @Override
+ public float getProgress() {
+ return 0;
+ }
+
+ @Override
+ public void onError(Throwable e) {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
new file mode 100644
index 0000000..78cdcde
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.taskcomm;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.protobuf.ByteString;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
+import org.apache.tez.dag.app.TezTestServiceCommunicator;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto;
+import org.apache.tez.util.ProtoConverters;
+
+
+public class TezTestServiceTaskCommunicatorImpl extends TezTaskCommunicatorImpl {
+
+ private static final Log LOG = LogFactory.getLog(TezTestServiceTaskCommunicatorImpl.class);
+
+ private final TezTestServiceCommunicator communicator;
+ private final SubmitWorkRequestProto BASE_SUBMIT_WORK_REQUEST;
+ private final ConcurrentMap<String, ByteBuffer> credentialMap;
+
+ public TezTestServiceTaskCommunicatorImpl(
+ TaskCommunicatorContext taskCommunicatorContext) {
+ super(taskCommunicatorContext);
+ // TODO Maybe make this configurable
+ this.communicator = new TezTestServiceCommunicator(3);
+
+ SubmitWorkRequestProto.Builder baseBuilder = SubmitWorkRequestProto.newBuilder();
+
+ // TODO Avoid reading this from the environment
+ baseBuilder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
+ baseBuilder.setApplicationIdString(
+ taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString());
+ baseBuilder
+ .setAppAttemptNumber(taskCommunicatorContext.getApplicationAttemptId().getAttemptId());
+ baseBuilder.setTokenIdentifier(getTokenIdentifier());
+
+ BASE_SUBMIT_WORK_REQUEST = baseBuilder.build();
+
+ credentialMap = new ConcurrentHashMap<String, ByteBuffer>();
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ this.communicator.init(conf);
+ }
+
+ @Override
+ public void serviceStart() {
+ super.serviceStart();
+ this.communicator.start();
+ }
+
+ @Override
+ public void serviceStop() {
+ super.serviceStop();
+ }
+
+
+ @Override
+ public void registerRunningContainer(ContainerId containerId, String hostname, int port) {
+ super.registerRunningContainer(containerId, hostname, port);
+ }
+
+ @Override
+ public void registerContainerEnd(ContainerId containerId) {
+ super.registerContainerEnd(containerId);
+ }
+
+ @Override
+ public void registerRunningTaskAttempt(final ContainerId containerId, final TaskSpec taskSpec,
+ Map<String, LocalResource> additionalResources,
+ Credentials credentials,
+ boolean credentialsChanged) {
+ super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials,
+ credentialsChanged);
+ SubmitWorkRequestProto requestProto = null;
+ try {
+ requestProto = constructSubmitWorkRequest(containerId, taskSpec);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to construct request", e);
+ }
+ ContainerInfo containerInfo = getContainerInfo(containerId);
+ String host;
+ int port;
+ if (containerInfo != null) {
+ synchronized (containerInfo) {
+ host = containerInfo.host;
+ port = containerInfo.port;
+ }
+ } else {
+ // TODO Handle this properly
+ throw new RuntimeException("ContainerInfo not found for container: " + containerId +
+ ", while trying to launch task: " + taskSpec.getTaskAttemptID());
+ }
+ communicator.submitWork(requestProto, host, port,
+ new TezTestServiceCommunicator.ExecuteRequestCallback<SubmitWorkResponseProto>() {
+ @Override
+ public void setResponse(SubmitWorkResponseProto response) {
+ LOG.info("Successfully launched task: " + taskSpec.getTaskAttemptID());
+ getTaskCommunicatorContext()
+ .taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId);
+ }
+
+ @Override
+ public void indicateError(Throwable t) {
+ // TODO Handle this error. This is where an API on the context to indicate failure / rejection comes in.
+ LOG.info("Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
+ containerId, t);
+ }
+ });
+ }
+
+ @Override
+ public void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID) {
+ super.unregisterRunningTaskAttempt(taskAttemptID);
+ // Nothing else to do for now. The push API in the test does not support termination of a running task
+ }
+
+ private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId,
+ TaskSpec taskSpec) throws
+ IOException {
+ SubmitWorkRequestProto.Builder builder =
+ SubmitWorkRequestProto.newBuilder(BASE_SUBMIT_WORK_REQUEST);
+ builder.setContainerIdString(containerId.toString());
+ builder.setAmHost(getAddress().getHostName());
+ builder.setAmPort(getAddress().getPort());
+ Credentials taskCredentials = new Credentials();
+ // Credentials can change across DAGs. Ideally construct only once per DAG.
+ taskCredentials.addAll(getTaskCommunicatorContext().getCredentials());
+
+ ByteBuffer credentialsBinary = credentialMap.get(taskSpec.getDAGName());
+ if (credentialsBinary == null) {
+ credentialsBinary = serializeCredentials(getTaskCommunicatorContext().getCredentials());
+ credentialMap.putIfAbsent(taskSpec.getDAGName(), credentialsBinary.duplicate());
+ } else {
+ credentialsBinary = credentialsBinary.duplicate();
+ }
+ builder.setCredentialsBinary(ByteString.copyFrom(credentialsBinary));
+ builder.setTaskSpec(ProtoConverters.convertTaskSpecToProto(taskSpec));
+ return builder.build();
+ }
+
+ private ByteBuffer serializeCredentials(Credentials credentials) throws IOException {
+ Credentials containerCredentials = new Credentials();
+ containerCredentials.addAll(credentials);
+ DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
+ containerCredentials.writeTokenStorageToStream(containerTokens_dob);
+ ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0,
+ containerTokens_dob.getLength());
+ return containerCredentialsBuffer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java
new file mode 100644
index 0000000..2bca4ed
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service;
+
+import java.io.IOException;
+
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
+
+public interface ContainerRunner {
+
+ void queueContainer(RunContainerRequestProto request) throws IOException;
+ void submitWork(SubmitWorkRequestProto request) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
new file mode 100644
index 0000000..f47bd67
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.tez.service.impl.TezTestService;
+
+public class MiniTezTestServiceCluster extends AbstractService {
+
+ private static final Log LOG = LogFactory.getLog(MiniTezTestServiceCluster.class);
+
+ private final File testWorkDir;
+ private final long availableMemory;
+ private final int numExecutorsPerService;
+ private final String[] localDirs;
+ private final Configuration clusterSpecificConfiguration = new Configuration(false);
+
+ private TezTestService tezTestService;
+
+ public static MiniTezTestServiceCluster create(String clusterName, int numExecutorsPerService, long availableMemory, int numLocalDirs) {
+ return new MiniTezTestServiceCluster(clusterName, numExecutorsPerService, availableMemory, numLocalDirs);
+ }
+
+ // TODO Add support for multiple instances
+ private MiniTezTestServiceCluster(String clusterName, int numExecutorsPerService, long availableMemory, int numLocalDirs) {
+ super(clusterName + "_TezTestServerCluster");
+ Preconditions.checkArgument(numExecutorsPerService > 0);
+ Preconditions.checkArgument(availableMemory > 0);
+ Preconditions.checkArgument(numLocalDirs > 0);
+ String clusterNameTrimmed = clusterName.replace("$", "") + "_TezTestServerCluster";
+ File targetWorkDir = new File("target", clusterNameTrimmed);
+ try {
+ FileContext.getLocalFSFileContext().delete(
+ new Path(targetWorkDir.getAbsolutePath()), true);
+ } catch (Exception e) {
+ LOG.warn("Could not cleanup test workDir: " + targetWorkDir, e);
+ throw new RuntimeException("Could not cleanup test workDir: " + targetWorkDir, e);
+ }
+
+ if (Shell.WINDOWS) {
+ // The test working directory can exceed the maximum path length supported
+ // by some Windows APIs and cmd.exe (260 characters). To work around this,
+ // create a symlink in temporary storage with a much shorter path,
+ // targeting the full path to the test working directory. Then, use the
+ // symlink as the test working directory.
+ String targetPath = targetWorkDir.getAbsolutePath();
+ File link = new File(System.getProperty("java.io.tmpdir"),
+ String.valueOf(System.currentTimeMillis()));
+ String linkPath = link.getAbsolutePath();
+
+ try {
+ FileContext.getLocalFSFileContext().delete(new Path(linkPath), true);
+ } catch (IOException e) {
+ throw new YarnRuntimeException("could not cleanup symlink: " + linkPath, e);
+ }
+
+ // Guarantee target exists before creating symlink.
+ targetWorkDir.mkdirs();
+
+ Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
+ Shell.getSymlinkCommand(targetPath, linkPath));
+ try {
+ shexec.execute();
+ } catch (IOException e) {
+ throw new YarnRuntimeException(String.format(
+ "failed to create symlink from %s to %s, shell output: %s", linkPath,
+ targetPath, shexec.getOutput()), e);
+ }
+
+ this.testWorkDir = link;
+ } else {
+ this.testWorkDir = targetWorkDir;
+ }
+ this.numExecutorsPerService = numExecutorsPerService;
+ this.availableMemory = availableMemory;
+
+ // Setup Local Dirs
+ localDirs = new String[numLocalDirs];
+ for (int i = 0 ; i < numLocalDirs ; i++) {
+ File f = new File(testWorkDir, "localDir");
+ f.mkdirs();
+ LOG.info("Created localDir: " + f.getAbsolutePath());
+ localDirs[i] = f.getAbsolutePath();
+ }
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ tezTestService = new TezTestService(conf, numExecutorsPerService, availableMemory, localDirs);
+ tezTestService.init(conf);
+
+ }
+
+ @Override
+ public void serviceStart() {
+ tezTestService.start();
+
+ clusterSpecificConfiguration.set(TezTestServiceConfConstants.TEZ_TEST_SERVICE_HOSTS,
+ getServiceAddress().getHostName());
+ clusterSpecificConfiguration.setInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT,
+ getServiceAddress().getPort());
+
+ clusterSpecificConfiguration.setInt(
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE,
+ numExecutorsPerService);
+ clusterSpecificConfiguration.setLong(
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB, availableMemory);
+ }
+
+ @Override
+ public void serviceStop() {
+ tezTestService.stop();
+ }
+
+ /**
+ * return the address at which the service is listening
+ * @return host:port
+ */
+ public InetSocketAddress getServiceAddress() {
+ Preconditions.checkState(getServiceState() == STATE.STARTED);
+ return tezTestService.getListenerAddress();
+ }
+
+ public int getShufflePort() {
+ Preconditions.checkState(getServiceState() == STATE.STARTED);
+ return tezTestService.getShufflePort();
+ }
+
+ public Configuration getClusterSpecificConfiguration() {
+ Preconditions.checkState(getServiceState() == STATE.STARTED);
+ return clusterSpecificConfiguration;
+ }
+
+ // Mainly for verification
+ public int getNumSubmissions() {
+ return tezTestService.getNumSubmissions();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java
new file mode 100644
index 0000000..bf4a5bd
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service;
+
+public class TezTestServiceConfConstants {
+
+ private static final String TEZ_TEST_SERVICE_PREFIX = "tez.test.service.";
+
+ /** Number of executors per instance - used by the scheduler */
+ public static final String TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE = TEZ_TEST_SERVICE_PREFIX + "num.executors.per-instance";
+
+ /** Memory available per instance - used by the scheduler */
+ public static final String TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB = TEZ_TEST_SERVICE_PREFIX + "memory.per.instance.mb";
+
+ /** CPUs available per instance - used by the scheduler */
+ public static final String TEZ_TEST_SERVICE_VCPUS_PER_INSTANCE = TEZ_TEST_SERVICE_PREFIX + "vcpus.per.instance";
+
+
+ /** Hosts on which the service is running. Currently assuming a single port for all instances */
+ public static final String TEZ_TEST_SERVICE_HOSTS = TEZ_TEST_SERVICE_PREFIX + "hosts";
+
+ /** Port on which the Service(s) listen. Current a single port for all instances */
+ public static final String TEZ_TEST_SERVICE_RPC_PORT = TEZ_TEST_SERVICE_PREFIX + "rpc.port";
+
+ /** Number of threads to use in the AM to communicate with the external service */
+ public static final String TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS = TEZ_TEST_SERVICE_PREFIX + "communicator.num.threads";
+ public static final int TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT = 2;
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceProtocolBlockingPB.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceProtocolBlockingPB.java
new file mode 100644
index 0000000..1108f72
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceProtocolBlockingPB.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+
+@ProtocolInfo(protocolName = "org.apache.tez.service.TezTestServiceProtocolBlockingPB", protocolVersion = 1)
+public interface TezTestServiceProtocolBlockingPB extends TezTestServiceProtocolProtos.TezTestServiceProtocol.BlockingInterface {
+}
\ No newline at end of file
[41/50] [abbrv] tez git commit: TEZ-2131. Add additional tests for
tasks running in the AM. (sseth)
Posted by ss...@apache.org.
TEZ-2131. Add additional tests for tasks running in the AM. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/86ff66ae
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/86ff66ae
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/86ff66ae
Branch: refs/heads/TEZ-2003
Commit: 86ff66aeed148d43bc12bbe1151eb3e38dd212b0
Parents: 66d8892
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Feb 20 17:23:18 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:47:32 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../org/apache/tez/tests/TestExternalTezServices.java | 10 +++++++---
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/86ff66ae/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 76496c9..4b0a12b 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -6,5 +6,6 @@ ALL CHANGES:
TEZ-2122. Setup pluggable components at AM/Vertex level.
TEZ-2123. Fix component managers to use pluggable components. (Enable hybrid mode)
TEZ-2125. Create a task communicator for local mode. Allow tasks to run in the AM.
+ TEZ-2131. Add additional tests for tasks running in the AM.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/86ff66ae/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
index 01c2080..0ec972b 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
@@ -40,7 +40,6 @@ import org.apache.tez.service.MiniTezTestServiceCluster;
import org.apache.tez.test.MiniTezCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
public class TestExternalTezServices {
@@ -229,6 +228,13 @@ public class TestExternalTezServices {
}
@Test(timeout = 60000)
+ public void testAllInAM() throws Exception {
+ int expectedExternalSubmissions = 0; // All in AM
+ runJoinValidate("AllInAM", expectedExternalSubmissions, PROPS_IN_AM,
+ PROPS_IN_AM, PROPS_IN_AM);
+ }
+
+ @Test(timeout = 60000)
public void testMixed1() throws Exception { // M-ExtService, R-containers
int expectedExternalSubmissions = 4 + 0; //4 for 4 src files, 0 for num reducers.
runJoinValidate("Mixed1", expectedExternalSubmissions, PROPS_EXT_SERVICE_PUSH,
@@ -263,8 +269,6 @@ public class TestExternalTezServices {
PROPS_EXT_SERVICE_PUSH, PROPS_IN_AM);
}
-
- @Ignore // Re-activate this after the AM registers the shuffle token with the launcher.
@Test(timeout = 60000)
public void testMixed6() throws Exception { // M - AM, R - Service
int expectedExternalSubmissions = 0 + 3; // 3 for R in service
[13/50] [abbrv] tez git commit: TEZ-2317. Event processing backlog
can result in task failures for short tasks (bikas)
Posted by ss...@apache.org.
TEZ-2317. Event processing backlog can result in task failures for short tasks (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/e1968681
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/e1968681
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/e1968681
Branch: refs/heads/TEZ-2003
Commit: e1968681cee821103e0105e4948c4fc6dc949776
Parents: bfb34af
Author: Bikas Saha <bi...@apache.org>
Authored: Thu Apr 16 11:42:09 2015 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Thu Apr 16 11:42:09 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../apache/tez/dag/app/dag/impl/TaskImpl.java | 15 +++++++++++--
.../tez/dag/app/dag/impl/TestTaskImpl.java | 22 ++++++++++++++++++++
.../api/events/TaskStatusUpdateEvent.java | 2 ++
4 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/e1968681/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a41a724..ce9bfe8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -288,6 +288,8 @@ TEZ-UI CHANGES (TEZ-8):
Release 0.5.4: Unreleased
ALL CHANGES:
+ TEZ-2317. Event processing backlog can result in task failures for short
+ tasks
TEZ-2289. ATSHistoryLoggingService can generate ArrayOutOfBoundsException.
TEZ-2257. Fix potential NPEs in TaskReporter.
TEZ-2192. Relocalization does not check for source.
http://git-wip-us.apache.org/repos/asf/tez/blob/e1968681/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 10a688f..a1eed07 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -715,7 +715,18 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
public boolean canCommit(TezTaskAttemptID taskAttemptID) {
writeLock.lock();
try {
- if (getState() != TaskState.RUNNING) {
+ TaskState state = getState();
+ if (state == TaskState.SCHEDULED) {
+ // the actual running task ran and is done and asking for commit. we are still stuck
+ // in the scheduled state which indicates a backlog in event processing. lets wait for the
+ // backlog to clear. returning false will make the attempt come back to us.
+ LOG.debug("Event processing delay. "
+ + "Attempt committing before state machine transitioned to running : Task {}", taskId);
+ return false;
+ }
+ // at this point the attempt is no longer in scheduled state or else we would still
+ // have been in scheduled state in task impl.
+ if (state != TaskState.RUNNING) {
LOG.info("Task not running. Issuing kill to bad commit attempt " + taskAttemptID);
eventHandler.handle(new TaskAttemptEventKillRequest(taskAttemptID
, "Task not running. Bad attempt.", TaskAttemptTerminationCause.TERMINATED_ORPHANED));
@@ -758,7 +769,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
writeLock.unlock();
}
}
-
+
TaskAttemptImpl createAttempt(int attemptNumber) {
return new TaskAttemptImpl(getTaskId(), attemptNumber, eventHandler,
taskAttemptListener, conf, clock, taskHeartbeatHandler, appContext,
http://git-wip-us.apache.org/repos/asf/tez/blob/e1968681/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 1c4b319..9509df4 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -467,6 +467,28 @@ public class TestTaskImpl {
assertTaskSucceededState();
}
+
+
+ @Test(timeout = 5000)
+ public void testEventBacklogDuringTaskAttemptCommit() {
+ TezTaskID taskId = getNewTaskID();
+ scheduleTaskAttempt(taskId);
+ assertEquals(TaskState.SCHEDULED, mockTask.getState());
+ // simulate
+ // task in scheduled state due to event backlog - real task done and calling canCommit
+ assertFalse("Commit should return false to make running task wait",
+ mockTask.canCommit(mockTask.getLastAttempt().getID()));
+ launchTaskAttempt(mockTask.getLastAttempt().getID());
+ updateAttemptState(mockTask.getLastAttempt(), TaskAttemptState.RUNNING);
+ assertTrue("Task state in AM is running now. Can commit.",
+ mockTask.canCommit(mockTask.getLastAttempt().getID()));
+
+ updateAttemptState(mockTask.getLastAttempt(), TaskAttemptState.SUCCEEDED);
+ mockTask.handle(new TaskEventTAUpdate(mockTask.getLastAttempt().getID(),
+ TaskEventType.T_ATTEMPT_SUCCEEDED));
+
+ assertTaskSucceededState();
+ }
@Test(timeout = 5000)
http://git-wip-us.apache.org/repos/asf/tez/blob/e1968681/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java
index 875a345..6465bed 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java
@@ -66,6 +66,8 @@ public class TaskStatusUpdateEvent extends Event implements Writable {
if (statistics != null) {
out.writeBoolean(true);
statistics.write(out);
+ } else {
+ out.writeBoolean(false);
}
}
[30/50] [abbrv] tez git commit: TEZ-2006. Task communication plane
needs to be pluggable. (sseth)
Posted by ss...@apache.org.
TEZ-2006. Task communication plane needs to be pluggable. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c3c22855
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c3c22855
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c3c22855
Branch: refs/heads/TEZ-2003
Commit: c3c228558f16887fadf533ed2392d0dc49bcca53
Parents: 82ba10c
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Feb 12 11:25:45 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:44:31 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../apache/tez/dag/api/TaskCommunicator.java | 54 ++
.../tez/dag/api/TaskCommunicatorContext.java | 48 ++
.../tez/dag/api/TaskHeartbeatRequest.java | 63 +++
.../tez/dag/api/TaskHeartbeatResponse.java | 39 ++
.../java/org/apache/tez/dag/app/AppContext.java | 3 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 5 +
.../dag/app/TaskAttemptListenerImpTezDag.java | 499 +++++++------------
.../tez/dag/app/TezTaskCommunicatorImpl.java | 474 ++++++++++++++++++
.../app/launcher/LocalContainerLauncher.java | 10 +-
.../tez/dag/app/rm/container/AMContainer.java | 3 +-
.../rm/container/AMContainerEventAssignTA.java | 2 +
.../dag/app/rm/container/AMContainerImpl.java | 1 +
.../apache/tez/dag/app/MockDAGAppMaster.java | 25 +-
.../app/TestTaskAttemptListenerImplTezDag.java | 81 +--
15 files changed, 954 insertions(+), 354 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 1822fcb..d7e4be5 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -1,4 +1,5 @@
ALL CHANGES:
TEZ-2019. Temporarily allow the scheduler and launcher to be specified via configuration.
+ TEZ-2006. Task communication plane needs to be pluggable.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
new file mode 100644
index 0000000..97f9c16
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.net.InetSocketAddress;
+import java.util.Map;
+
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+
+// TODO TEZ-2003 Move this into the tez-api module
+public abstract class TaskCommunicator extends AbstractService {
+ public TaskCommunicator(String name) {
+ super(name);
+ }
+
+ // TODO TEZ-2003 Ideally, don't expose YARN containerId; instead expose a Tez specific construct.
+ // TODO When talking to an external service, this plugin implementer may need access to a host:port
+ public abstract void registerRunningContainer(ContainerId containerId, String hostname, int port);
+
+ // TODO TEZ-2003 Ideally, don't expose YARN containerId; instead expose a Tez specific construct.
+ public abstract void registerContainerEnd(ContainerId containerId);
+
+ // TODO TEZ-2003 TaskSpec breakup into a clean interface
+ // TODO TEZ-2003 Add support for priority
+ public abstract void registerRunningTaskAttempt(ContainerId containerId, TaskSpec taskSpec,
+ Map<String, LocalResource> additionalResources,
+ Credentials credentials,
+ boolean credentialsChanged);
+
+ // TODO TEZ-2003 Remove reference to TaskAttemptID
+ public abstract void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID);
+
+ // TODO TEZ-2003 This doesn't necessarily belong here. A server may not start within the AM.
+ public abstract InetSocketAddress getAddress();
+
+ // TODO Eventually. Add methods here to support preemption of tasks.
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
new file mode 100644
index 0000000..9b2d889
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+
+
+// Do not make calls into this from within a held lock.
+
+// TODO TEZ-2003 Move this into the tez-api module
+public interface TaskCommunicatorContext {
+
+ // TODO TEZ-2003 Add signalling back into this to indicate errors - e.g. Container unregsitered, task no longer running, etc.
+
+ // TODO TEZ-2003 Maybe add book-keeping as a helper library, instead of each impl tracking container to task etc.
+
+ ApplicationAttemptId getApplicationAttemptId();
+ Credentials getCredentials();
+
+ // TODO TEZ-2003 Move to vertex, taskIndex, version
+ boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException;
+
+ TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request) throws IOException, TezException;
+
+ boolean isKnownContainer(ContainerId containerId);
+
+ // TODO TEZ-2003 Move to vertex, taskIndex, version
+ void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId);
+
+ // TODO Eventually Add methods to report availability stats to the scheduler.
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatRequest.java
new file mode 100644
index 0000000..f6bc8f0
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatRequest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.util.List;
+
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TezEvent;
+
+// TODO TEZ-2003 Move this into the tez-api module
+public class TaskHeartbeatRequest {
+
+ // TODO TEZ-2003 Ideally containerIdentifier should not be part of the request.
+ // Replace with a task lookup - vertex name + task index
+ private final String containerIdentifier;
+ // TODO TEZ-2003 Get rid of the task attemptId reference if possible
+ private final TezTaskAttemptID taskAttemptId;
+ private final List<TezEvent> events;
+ private final int startIndex;
+ private final int maxEvents;
+
+
+ public TaskHeartbeatRequest(String containerIdentifier, TezTaskAttemptID taskAttemptId, List<TezEvent> events, int startIndex,
+ int maxEvents) {
+ this.containerIdentifier = containerIdentifier;
+ this.taskAttemptId = taskAttemptId;
+ this.events = events;
+ this.startIndex = startIndex;
+ this.maxEvents = maxEvents;
+ }
+
+ public String getContainerIdentifier() {
+ return containerIdentifier;
+ }
+
+ public TezTaskAttemptID getTaskAttemptId() {
+ return taskAttemptId;
+ }
+
+ public List<TezEvent> getEvents() {
+ return events;
+ }
+
+ public int getStartIndex() {
+ return startIndex;
+ }
+
+ public int getMaxEvents() {
+ return maxEvents;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatResponse.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatResponse.java
new file mode 100644
index 0000000..c82a743
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatResponse.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.util.List;
+
+import org.apache.tez.runtime.api.impl.TezEvent;
+
+// TODO TEZ-2003 Move this into the tez-api module
+public class TaskHeartbeatResponse {
+
+ private final boolean shouldDie;
+ private List<TezEvent> events;
+
+ public TaskHeartbeatResponse(boolean shouldDie, List<TezEvent> events) {
+ this.shouldDie = shouldDie;
+ this.events = events;
+ }
+
+ public boolean isShouldDie() {
+ return shouldDie;
+ }
+
+ public List<TezEvent> getEvents() {
+ return events;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index 4781784..37f7624 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -106,4 +107,6 @@ public interface AppContext {
String[] getLocalDirs();
String getAMUser();
+
+ Credentials getAppCredentials();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index a6d6c69..f9ec44d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -1474,6 +1474,11 @@ public class DAGAppMaster extends AbstractService {
}
@Override
+ public Credentials getAppCredentials() {
+ return amCredentials;
+ }
+
+ @Override
public Map<ApplicationAccessType, String> getApplicationACLs() {
if (getServiceState() != STATE.STARTED) {
throw new TezUncheckedException(
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index b64283b..68b5a75 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -18,228 +18,209 @@
package org.apache.tez.dag.app;
import java.io.IOException;
-import java.net.InetAddress;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.net.InetSocketAddress;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TaskCommunicator;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.api.TaskHeartbeatResponse;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.common.ContainerContext;
-import org.apache.tez.common.ContainerTask;
-import org.apache.tez.common.TezConverterUtils;
-import org.apache.tez.common.TezLocalResource;
-import org.apache.tez.common.TezTaskUmbilicalProtocol;
-import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TaskHeartbeatRequest;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
+import org.apache.tez.dag.app.rm.TaskSchedulerService;
import org.apache.tez.dag.app.rm.container.AMContainerTask;
-import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.runtime.api.impl.TezEvent;
-import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
-import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
import org.apache.tez.common.security.JobTokenSecretManager;
-import com.google.common.collect.Maps;
@SuppressWarnings("unchecked")
+@InterfaceAudience.Private
public class TaskAttemptListenerImpTezDag extends AbstractService implements
- TezTaskUmbilicalProtocol, TaskAttemptListener {
-
- private static final ContainerTask TASK_FOR_INVALID_JVM = new ContainerTask(
- null, true, null, null, false);
+ TaskAttemptListener, TaskCommunicatorContext {
private static final Logger LOG = LoggerFactory
.getLogger(TaskAttemptListenerImpTezDag.class);
private final AppContext context;
+ private TaskCommunicator taskCommunicator;
protected final TaskHeartbeatHandler taskHeartbeatHandler;
protected final ContainerHeartbeatHandler containerHeartbeatHandler;
- private final JobTokenSecretManager jobTokenSecretManager;
- private InetSocketAddress address;
- private Server server;
-
- static class ContainerInfo {
- ContainerInfo() {
- this.lastReponse = null;
- this.lastRequestId = 0;
- this.amContainerTask = null;
- this.taskPulled = false;
+
+ private final TaskHeartbeatResponse RESPONSE_SHOULD_DIE = new TaskHeartbeatResponse(true, null);
+
+ private final ConcurrentMap<TezTaskAttemptID, ContainerId> registeredAttempts =
+ new ConcurrentHashMap<TezTaskAttemptID, ContainerId>();
+ private final ConcurrentMap<ContainerId, ContainerInfo> registeredContainers =
+ new ConcurrentHashMap<ContainerId, ContainerInfo>();
+
+ // Defined primarily to work around ConcurrentMaps not accepting null values
+ private static final class ContainerInfo {
+ TezTaskAttemptID taskAttemptId;
+ ContainerInfo(TezTaskAttemptID taskAttemptId) {
+ this.taskAttemptId = taskAttemptId;
}
- long lastRequestId;
- TezHeartbeatResponse lastReponse;
- AMContainerTask amContainerTask;
- boolean taskPulled;
}
- private ConcurrentMap<TezTaskAttemptID, ContainerId> attemptToInfoMap =
- new ConcurrentHashMap<TezTaskAttemptID, ContainerId>();
+ private static final ContainerInfo NULL_CONTAINER_INFO = new ContainerInfo(null);
- private ConcurrentHashMap<ContainerId, ContainerInfo> registeredContainers =
- new ConcurrentHashMap<ContainerId, ContainerInfo>();
public TaskAttemptListenerImpTezDag(AppContext context,
- TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
- JobTokenSecretManager jobTokenSecretManager) {
+ TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
+ // TODO TEZ-2003 pre-merge. Remove reference to JobTokenSecretManager.
+ JobTokenSecretManager jobTokenSecretManager) {
super(TaskAttemptListenerImpTezDag.class.getName());
this.context = context;
- this.jobTokenSecretManager = jobTokenSecretManager;
this.taskHeartbeatHandler = thh;
this.containerHeartbeatHandler = chh;
+ this.taskCommunicator = new TezTaskCommunicatorImpl(this);
}
@Override
- public void serviceStart() {
- startRpcServer();
- }
-
- protected void startRpcServer() {
- Configuration conf = getConfig();
- if (!conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT)) {
- try {
- server = new RPC.Builder(conf)
- .setProtocol(TezTaskUmbilicalProtocol.class)
- .setBindAddress("0.0.0.0")
- .setPort(0)
- .setInstance(this)
- .setNumHandlers(
- conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
- TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT))
- .setSecretManager(jobTokenSecretManager).build();
-
- // Enable service authorization?
- if (conf.getBoolean(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
- false)) {
- refreshServiceAcls(conf, new TezAMPolicyProvider());
- }
-
- server.start();
- this.address = NetUtils.getConnectAddress(server);
- } catch (IOException e) {
- throw new TezUncheckedException(e);
- }
+ public void serviceInit(Configuration conf) {
+ String taskCommClassName = conf.get(TezConfiguration.TEZ_AM_TASK_COMMUNICATOR_CLASS);
+ if (taskCommClassName == null) {
+ LOG.info("Using Default Task Communicator");
+ this.taskCommunicator = new TezTaskCommunicatorImpl(this);
} else {
+ LOG.info("Using TaskCommunicator: " + taskCommClassName);
+ Class<? extends TaskCommunicator> taskCommClazz = (Class<? extends TaskCommunicator>) ReflectionUtils
+ .getClazz(taskCommClassName);
try {
- this.address = new InetSocketAddress(InetAddress.getLocalHost(), 0);
- } catch (UnknownHostException e) {
+ Constructor<? extends TaskCommunicator> ctor = taskCommClazz.getConstructor(TaskCommunicatorContext.class);
+ ctor.setAccessible(true);
+ this.taskCommunicator = ctor.newInstance(this);
+ } catch (NoSuchMethodException e) {
+ throw new TezUncheckedException(e);
+ } catch (InvocationTargetException e) {
+ throw new TezUncheckedException(e);
+ } catch (InstantiationException e) {
+ throw new TezUncheckedException(e);
+ } catch (IllegalAccessException e) {
throw new TezUncheckedException(e);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Not starting TaskAttemptListener RPC in LocalMode");
}
}
}
- void refreshServiceAcls(Configuration configuration,
- PolicyProvider policyProvider) {
- this.server.refreshServiceAcl(configuration, policyProvider);
+ @Override
+ public void serviceStart() {
+ taskCommunicator.init(getConfig());
+ taskCommunicator.start();
}
@Override
public void serviceStop() {
- stopRpcServer();
- }
-
- protected void stopRpcServer() {
- if (server != null) {
- server.stop();
+ if (taskCommunicator != null) {
+ taskCommunicator.stop();
+ taskCommunicator = null;
}
}
- public InetSocketAddress getAddress() {
- return address;
- }
-
@Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return versionID;
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return context.getApplicationAttemptId();
}
@Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- return ProtocolSignature.getProtocolSignature(this, protocol,
- clientVersion, clientMethodsHash);
+ public Credentials getCredentials() {
+ return context.getAppCredentials();
}
@Override
- public ContainerTask getTask(ContainerContext containerContext)
- throws IOException {
+ public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request)
+ throws IOException, TezException {
+ ContainerId containerId = ConverterUtils.toContainerId(request
+ .getContainerIdentifier());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received heartbeat from container"
+ + ", request=" + request);
+ }
- ContainerTask task = null;
+ if (!registeredContainers.containsKey(containerId)) {
+ LOG.warn("Received task heartbeat from unknown container with id: " + containerId +
+ ", asking it to die");
+ return RESPONSE_SHOULD_DIE;
+ }
- if (containerContext == null || containerContext.getContainerIdentifier() == null) {
- LOG.info("Invalid task request with an empty containerContext or containerId");
- task = TASK_FOR_INVALID_JVM;
- } else {
- ContainerId containerId = ConverterUtils.toContainerId(containerContext
- .getContainerIdentifier());
+ // A heartbeat can come in anytime. The AM may have made a decision to kill a running task/container
+ // meanwhile. If the decision is processed through the pipeline before the heartbeat is processed,
+ // the heartbeat will be dropped. Otherwise the heartbeat will be processed - and the system
+ // know how to handle this - via FailedInputEvents for example (relevant only if the heartbeat has events).
+ // So - avoiding synchronization.
+
+ pingContainerHeartbeatHandler(containerId);
+ List<TezEvent> outEvents = null;
+ TezTaskAttemptID taskAttemptID = request.getTaskAttemptId();
+ if (taskAttemptID != null) {
+ ContainerId containerIdFromMap = registeredAttempts.get(taskAttemptID);
+ if (containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
+ // This can happen when a task heartbeats. Meanwhile the container is unregistered.
+ // The information will eventually make it through to the plugin via a corresponding unregister.
+ // There's a race in that case between the unregister making it through, and this method returning.
+ // TODO TEZ-2003. An exception back is likely a better approach than sending a shouldDie = true,
+ // so that the plugin can handle the scenario. Alternately augment the response with error codes.
+ // Error codes would be better than exceptions.
+ LOG.info("Attempt: " + taskAttemptID + " is not recognized for heartbeats");
+ return RESPONSE_SHOULD_DIE;
+ }
+
+ List<TezEvent> inEvents = request.getEvents();
if (LOG.isDebugEnabled()) {
- LOG.debug("Container with id: " + containerId + " asked for a task");
+ LOG.debug("Ping from " + taskAttemptID.toString() +
+ " events: " + (inEvents != null ? inEvents.size() : -1));
}
- if (!registeredContainers.containsKey(containerId)) {
- if(context.getAllContainers().get(containerId) == null) {
- LOG.info("Container with id: " + containerId
- + " is invalid and will be killed");
- } else {
- LOG.info("Container with id: " + containerId
- + " is valid, but no longer registered, and will be killed");
- }
- task = TASK_FOR_INVALID_JVM;
- } else {
- pingContainerHeartbeatHandler(containerId);
- task = getContainerTask(containerId);
- if (task == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No task current assigned to Container with id: " + containerId);
- }
- } else if (task == TASK_FOR_INVALID_JVM) {
- LOG.info("Container with id: " + containerId
- + " is valid, but no longer registered, and will be killed. Race condition.");
- } else {
- context.getEventHandler().handle(
- new TaskAttemptEventStartedRemotely(task.getTaskSpec()
- .getTaskAttemptID(), containerId, context
- .getApplicationACLs()));
- LOG.info("Container with id: " + containerId + " given task: "
- + task.getTaskSpec().getTaskAttemptID());
- }
+ if (inEvents != null && !inEvents.isEmpty()) {
+ TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
+ context.getEventHandler().handle(
+ new VertexEventRouteEvent(vertexId, inEvents));
}
+ taskHeartbeatHandler.pinged(taskAttemptID);
+ outEvents = context
+ .getCurrentDAG()
+ .getVertex(taskAttemptID.getTaskID().getVertexID())
+ .getTask(taskAttemptID.getTaskID())
+ .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(),
+ request.getMaxEvents());
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("getTask returning task: " + task);
- }
- return task;
+ return new TaskHeartbeatResponse(false, outEvents);
+ }
+
+ @Override
+ public boolean isKnownContainer(ContainerId containerId) {
+ return context.getAllContainers().get(containerId) != null;
+ }
+
+ @Override
+ public void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId) {
+ context.getEventHandler().handle(new TaskAttemptEventStartedRemotely(taskAttemptID, containerId, null));
+ pingContainerHeartbeatHandler(containerId);
}
/**
* Child checking whether it can commit.
- *
+ * <p/>
* <br/>
* Repeatedly polls the ApplicationMaster whether it
* {@link Task#canCommit(TezTaskAttemptID)} This is * a legacy from the
@@ -262,25 +243,12 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
@Override
- public void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
- ContainerId containerId = attemptToInfoMap.get(attemptId);
- if(containerId == null) {
- LOG.warn("Unregister task attempt: " + attemptId + " from unknown container");
- return;
- }
- ContainerInfo containerInfo = registeredContainers.get(containerId);
- if(containerInfo == null) {
- LOG.warn("Unregister task attempt: " + attemptId +
- " from non-registered container: " + containerId);
- return;
- }
- synchronized (containerInfo) {
- containerInfo.amContainerTask = null;
- attemptToInfoMap.remove(attemptId);
- }
-
+ public InetSocketAddress getAddress() {
+ return taskCommunicator.getAddress();
}
+ // The TaskAttemptListener register / unregister methods in this class are not thread safe.
+ // The Tez framework should not invoke these methods from multiple threads.
@Override
public void dagComplete(DAG dag) {
// TODO TEZ-2335. Cleanup TaskHeartbeat handler structures.
@@ -300,50 +268,82 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
@Override
public void registerRunningContainer(ContainerId containerId) {
if (LOG.isDebugEnabled()) {
- LOG.debug("ContainerId: " + containerId
- + " registered with TaskAttemptListener");
+ LOG.debug("ContainerId: " + containerId + " registered with TaskAttemptListener");
}
- ContainerInfo oldInfo = registeredContainers.put(containerId, new ContainerInfo());
- if(oldInfo != null) {
+ ContainerInfo oldInfo = registeredContainers.put(containerId, NULL_CONTAINER_INFO);
+ if (oldInfo != null) {
throw new TezUncheckedException(
"Multiple registrations for containerId: " + containerId);
}
+ NodeId nodeId = context.getAllContainers().get(containerId).getContainer().getNodeId();
+ taskCommunicator.registerRunningContainer(containerId, nodeId.getHost(), nodeId.getPort());
+ }
+
+ @Override
+ public void unregisterRunningContainer(ContainerId containerId) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unregistering Container from TaskAttemptListener: " + containerId);
+ }
+ ContainerInfo containerInfo = registeredContainers.remove(containerId);
+ if (containerInfo.taskAttemptId != null) {
+ registeredAttempts.remove(containerInfo.taskAttemptId);
+ }
+ taskCommunicator.registerContainerEnd(containerId);
}
@Override
public void registerTaskAttempt(AMContainerTask amContainerTask,
- ContainerId containerId) {
+ ContainerId containerId) {
ContainerInfo containerInfo = registeredContainers.get(containerId);
- if(containerInfo == null) {
+ if (containerInfo == null) {
throw new TezUncheckedException("Registering task attempt: "
+ amContainerTask.getTask().getTaskAttemptID() + " to unknown container: " + containerId);
}
- synchronized (containerInfo) {
- if(containerInfo.amContainerTask != null) {
- throw new TezUncheckedException("Registering task attempt: "
- + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
- + " with existing assignment to: " + containerInfo.amContainerTask.getTask().getTaskAttemptID());
- }
- containerInfo.amContainerTask = amContainerTask;
- containerInfo.taskPulled = false;
-
- ContainerId containerIdFromMap =
- attemptToInfoMap.put(amContainerTask.getTask().getTaskAttemptID(), containerId);
- if(containerIdFromMap != null) {
- throw new TezUncheckedException("Registering task attempt: "
- + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
- + " when already assigned to: " + containerIdFromMap);
- }
+ if (containerInfo.taskAttemptId != null) {
+ throw new TezUncheckedException("Registering task attempt: "
+ + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+ + " with existing assignment to: " +
+ containerInfo.taskAttemptId);
+ }
+
+ if (containerInfo.taskAttemptId != null) {
+ throw new TezUncheckedException("Registering task attempt: "
+ + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+ + " with existing assignment to: " +
+ containerInfo.taskAttemptId);
}
+
+ // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
+ registeredContainers.put(containerId, new ContainerInfo(amContainerTask.getTask().getTaskAttemptID()));
+
+ ContainerId containerIdFromMap = registeredAttempts.put(
+ amContainerTask.getTask().getTaskAttemptID(), containerId);
+ if (containerIdFromMap != null) {
+ throw new TezUncheckedException("Registering task attempt: "
+ + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+ + " when already assigned to: " + containerIdFromMap);
+ }
+ taskCommunicator.registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
+ amContainerTask.getAdditionalResources(), amContainerTask.getCredentials(),
+ amContainerTask.haveCredentialsChanged());
}
@Override
- public void unregisterRunningContainer(ContainerId containerId) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Unregistering Container from TaskAttemptListener: "
- + containerId);
+ public void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
+ ContainerId containerId = registeredAttempts.remove(attemptId);
+ if (containerId == null) {
+ LOG.warn("Unregister task attempt: " + attemptId + " from unknown container");
+ return;
+ }
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ if (containerInfo == null) {
+ LOG.warn("Unregister task attempt: " + attemptId +
+ " from non-registered container: " + containerId);
+ return;
}
- registeredContainers.remove(containerId);
+ // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
+ registeredContainers.put(containerId, NULL_CONTAINER_INFO);
+ taskCommunicator.unregisterRunningTaskAttempt(attemptId);
}
private void pingContainerHeartbeatHandler(ContainerId containerId) {
@@ -351,7 +351,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
private void pingContainerHeartbeatHandler(TezTaskAttemptID taskAttemptId) {
- ContainerId containerId = attemptToInfoMap.get(taskAttemptId);
+ ContainerId containerId = registeredAttempts.get(taskAttemptId);
if (containerId != null) {
containerHeartbeatHandler.pinged(containerId);
} else {
@@ -360,124 +360,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
}
- @Override
- public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request)
- throws IOException, TezException {
- ContainerId containerId = ConverterUtils.toContainerId(request
- .getContainerIdentifier());
- long requestId = request.getRequestId();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Received heartbeat from container"
- + ", request=" + request);
- }
-
- ContainerInfo containerInfo = registeredContainers.get(containerId);
- if(containerInfo == null) {
- LOG.warn("Received task heartbeat from unknown container with id: " + containerId +
- ", asking it to die");
- TezHeartbeatResponse response = new TezHeartbeatResponse();
- response.setLastRequestId(requestId);
- response.setShouldDie();
- return response;
- }
-
- synchronized (containerInfo) {
- pingContainerHeartbeatHandler(containerId);
-
- if(containerInfo.lastRequestId == requestId) {
- LOG.warn("Old sequenceId received: " + requestId
- + ", Re-sending last response to client");
- return containerInfo.lastReponse;
- }
-
- TezHeartbeatResponse response = new TezHeartbeatResponse();
- response.setLastRequestId(requestId);
-
- TezTaskAttemptID taskAttemptID = request.getCurrentTaskAttemptID();
- if (taskAttemptID != null) {
- ContainerId containerIdFromMap = attemptToInfoMap.get(taskAttemptID);
- if(containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
- throw new TezException("Attempt " + taskAttemptID
- + " is not recognized for heartbeat");
- }
-
- if(containerInfo.lastRequestId+1 != requestId) {
- throw new TezException("Container " + containerId
- + " has invalid request id. Expected: "
- + containerInfo.lastRequestId+1
- + " and actual: " + requestId);
- }
-
- List<TezEvent> inEvents = request.getEvents();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Ping from " + taskAttemptID.toString() +
- " events: " + (inEvents != null? inEvents.size() : -1));
- }
- if(inEvents!=null && !inEvents.isEmpty()) {
- TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
- context.getEventHandler().handle(
- new VertexEventRouteEvent(vertexId, inEvents));
- }
- taskHeartbeatHandler.pinged(taskAttemptID);
- List<TezEvent> outEvents = context
- .getCurrentDAG()
- .getVertex(taskAttemptID.getTaskID().getVertexID())
- .getTask(taskAttemptID.getTaskID())
- .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(),
- request.getMaxEvents());
- response.setEvents(outEvents);
- }
- containerInfo.lastRequestId = requestId;
- containerInfo.lastReponse = response;
- return response;
- }
- }
-
- private Map<String, TezLocalResource> convertLocalResourceMap(Map<String, LocalResource> ylrs)
- throws IOException {
- Map<String, TezLocalResource> tlrs = Maps.newHashMap();
- if (ylrs != null) {
- for (Entry<String, LocalResource> ylrEntry : ylrs.entrySet()) {
- TezLocalResource tlr;
- try {
- tlr = TezConverterUtils.convertYarnLocalResourceToTez(ylrEntry.getValue());
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- tlrs.put(ylrEntry.getKey(), tlr);
- }
- }
- return tlrs;
- }
-
- private ContainerTask getContainerTask(ContainerId containerId) throws IOException {
- ContainerTask containerTask = null;
- ContainerInfo containerInfo = registeredContainers.get(containerId);
- if (containerInfo == null) {
- // This can happen if an unregisterTask comes in after we've done the initial checks for
- // registered containers. (Race between getTask from the container, and a potential STOP_CONTAINER
- // from somewhere within the AM)
- // Implies that an un-registration has taken place and the container needs to be asked to die.
- LOG.info("Container with id: " + containerId
- + " is valid, but no longer registered, and will be killed");
- containerTask = TASK_FOR_INVALID_JVM;
- } else {
- synchronized (containerInfo) {
- if (containerInfo.amContainerTask != null) {
- if (!containerInfo.taskPulled) {
- containerInfo.taskPulled = true;
- AMContainerTask amContainerTask = containerInfo.amContainerTask;
- containerTask = new ContainerTask(amContainerTask.getTask(), false,
- convertLocalResourceMap(amContainerTask.getAdditionalResources()),
- amContainerTask.getCredentials(), amContainerTask.haveCredentialsChanged());
- } else {
- containerTask = null;
- }
- } else {
- containerTask = null;
- }
- }
- }
- return containerTask;
+ public TaskCommunicator getTaskCommunicator() {
+ return taskCommunicator;
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
new file mode 100644
index 0000000..5652937
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
@@ -0,0 +1,474 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.common.*;
+import org.apache.tez.common.ContainerContext;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.TaskCommunicator;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.api.TaskHeartbeatRequest;
+import org.apache.tez.dag.api.TaskHeartbeatResponse;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
+import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
+
+@InterfaceAudience.Private
+public class TezTaskCommunicatorImpl extends TaskCommunicator {
+
+ private static final Log LOG = LogFactory.getLog(TezTaskCommunicatorImpl.class);
+
+ private static final ContainerTask TASK_FOR_INVALID_JVM = new ContainerTask(
+ null, true, null, null, false);
+
+ private final TaskCommunicatorContext taskCommunicatorContext;
+
+ private final ConcurrentMap<ContainerId, ContainerInfo> registeredContainers =
+ new ConcurrentHashMap<ContainerId, ContainerInfo>();
+ private final ConcurrentMap<TaskAttempt, ContainerId> attemptToContainerMap =
+ new ConcurrentHashMap<TaskAttempt, ContainerId>();
+
+ private final TezTaskUmbilicalProtocol taskUmbilical;
+ private InetSocketAddress address;
+ private Server server;
+
+ private static final class ContainerInfo {
+
+ ContainerInfo(ContainerId containerId) {
+ this.containerId = containerId;
+ }
+
+ ContainerId containerId;
+ TezHeartbeatResponse lastResponse = null;
+ TaskSpec taskSpec = null;
+ long lastRequestId = 0;
+ Map<String, LocalResource> additionalLRs = null;
+ Credentials credentials = null;
+ boolean credentialsChanged = false;
+ boolean taskPulled = false;
+
+ void reset() {
+ taskSpec = null;
+ additionalLRs = null;
+ credentials = null;
+ credentialsChanged = false;
+ taskPulled = false;
+ }
+ }
+
+
+
+ /**
+ * Construct the service.
+ */
+ public TezTaskCommunicatorImpl(TaskCommunicatorContext taskCommunicatorContext) {
+ super(TezTaskCommunicatorImpl.class.getName());
+ this.taskCommunicatorContext = taskCommunicatorContext;
+ this.taskUmbilical = new TezTaskUmbilicalProtocolImpl();
+ }
+
+
+ @Override
+ public void serviceStart() {
+
+ startRpcServer();
+ }
+
+ @Override
+ public void serviceStop() {
+ stopRpcServer();
+ }
+
+ protected void startRpcServer() {
+ Configuration conf = getConfig();
+ if (!conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT)) {
+ try {
+ JobTokenSecretManager jobTokenSecretManager =
+ new JobTokenSecretManager();
+ Token<JobTokenIdentifier> sessionToken = TokenCache.getSessionToken(taskCommunicatorContext.getCredentials());
+ jobTokenSecretManager.addTokenForJob(
+ taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString(), sessionToken);
+
+ server = new RPC.Builder(conf)
+ .setProtocol(TezTaskUmbilicalProtocol.class)
+ .setBindAddress("0.0.0.0")
+ .setPort(0)
+ .setInstance(taskUmbilical)
+ .setNumHandlers(
+ conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
+ TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT))
+ .setSecretManager(jobTokenSecretManager).build();
+
+ // Enable service authorization?
+ if (conf.getBoolean(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
+ false)) {
+ refreshServiceAcls(conf, new TezAMPolicyProvider());
+ }
+
+ server.start();
+ this.address = NetUtils.getConnectAddress(server);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ } else {
+ try {
+ this.address = new InetSocketAddress(InetAddress.getLocalHost(), 0);
+ } catch (UnknownHostException e) {
+ throw new TezUncheckedException(e);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not starting TaskAttemptListener RPC in LocalMode");
+ }
+ }
+ }
+
+ protected void stopRpcServer() {
+ if (server != null) {
+ server.stop();
+ server = null;
+ }
+ }
+
+ private void refreshServiceAcls(Configuration configuration,
+ PolicyProvider policyProvider) {
+ this.server.refreshServiceAcl(configuration, policyProvider);
+ }
+
+ @Override
+ public void registerRunningContainer(ContainerId containerId, String host, int port) {
+ ContainerInfo oldInfo = registeredContainers.putIfAbsent(containerId, new ContainerInfo(containerId));
+ if (oldInfo != null) {
+ throw new TezUncheckedException("Multiple registrations for containerId: " + containerId);
+ }
+ }
+
+ @Override
+ public void registerContainerEnd(ContainerId containerId) {
+ ContainerInfo containerInfo = registeredContainers.remove(containerId);
+ if (containerInfo != null) {
+ synchronized(containerInfo) {
+ if (containerInfo.taskSpec != null && containerInfo.taskSpec.getTaskAttemptID() != null) {
+ attemptToContainerMap.remove(containerInfo.taskSpec.getTaskAttemptID());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void registerRunningTaskAttempt(ContainerId containerId, TaskSpec taskSpec,
+ Map<String, LocalResource> additionalResources,
+ Credentials credentials, boolean credentialsChanged) {
+
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ Preconditions.checkNotNull(containerInfo,
+ "Cannot register task attempt: " + taskSpec.getTaskAttemptID() + " to unknown container: " +
+ containerId);
+ synchronized (containerInfo) {
+ if (containerInfo.taskSpec != null) {
+ throw new TezUncheckedException(
+ "Cannot register task: " + taskSpec.getTaskAttemptID() + " to container: " +
+ containerId + " , with pre-existing assignment: " +
+ containerInfo.taskSpec.getTaskAttemptID());
+ }
+ containerInfo.taskSpec = taskSpec;
+ containerInfo.additionalLRs = additionalResources;
+ containerInfo.credentials = credentials;
+ containerInfo.credentialsChanged = credentialsChanged;
+ containerInfo.taskPulled = false;
+
+ ContainerId oldId = attemptToContainerMap.putIfAbsent(new TaskAttempt(taskSpec.getTaskAttemptID()), containerId);
+ if (oldId != null) {
+ throw new TezUncheckedException(
+ "Attempting to register an already registered taskAttempt with id: " +
+ taskSpec.getTaskAttemptID() + " to containerId: " + containerId +
+ ". Already registered to containerId: " + oldId);
+ }
+ }
+
+ }
+
+ @Override
+ public void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID) {
+ TaskAttempt taskAttempt = new TaskAttempt(taskAttemptID);
+ ContainerId containerId = attemptToContainerMap.remove(taskAttempt);
+ if(containerId == null) {
+ LOG.warn("Unregister task attempt: " + taskAttempt + " from unknown container");
+ return;
+ }
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ if (containerInfo == null) {
+ LOG.warn("Unregister task attempt: " + taskAttempt +
+ " from non-registered container: " + containerId);
+ return;
+ }
+ synchronized (containerInfo) {
+ containerInfo.reset();
+ attemptToContainerMap.remove(taskAttempt);
+ }
+ }
+
+ @Override
+ public InetSocketAddress getAddress() {
+ return address;
+ }
+
+ public TezTaskUmbilicalProtocol getUmbilical() {
+ return this.taskUmbilical;
+ }
+
+ private class TezTaskUmbilicalProtocolImpl implements TezTaskUmbilicalProtocol {
+
+ @Override
+ public ContainerTask getTask(ContainerContext containerContext) throws IOException {
+ ContainerTask task = null;
+ if (containerContext == null || containerContext.getContainerIdentifier() == null) {
+ LOG.info("Invalid task request with an empty containerContext or containerId");
+ task = TASK_FOR_INVALID_JVM;
+ } else {
+ ContainerId containerId = ConverterUtils.toContainerId(containerContext
+ .getContainerIdentifier());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Container with id: " + containerId + " asked for a task");
+ }
+ task = getContainerTask(containerId);
+ if (task != null && !task.shouldDie()) {
+ taskCommunicatorContext
+ .taskStartedRemotely(task.getTaskSpec().getTaskAttemptID(), containerId);
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getTask returning task: " + task);
+ }
+ return task;
+ }
+
+ @Override
+ public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
+ return taskCommunicatorContext.canCommit(taskAttemptId);
+ }
+
+ @Override
+ public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException,
+ TezException {
+ ContainerId containerId = ConverterUtils.toContainerId(request.getContainerIdentifier());
+ long requestId = request.getRequestId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received heartbeat from container"
+ + ", request=" + request);
+ }
+
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ if (containerInfo == null) {
+ LOG.warn("Received task heartbeat from unknown container with id: " + containerId +
+ ", asking it to die");
+ TezHeartbeatResponse response = new TezHeartbeatResponse();
+ response.setLastRequestId(requestId);
+ response.setShouldDie();
+ return response;
+ }
+
+ synchronized (containerInfo) {
+ if (containerInfo.lastRequestId == requestId) {
+ LOG.warn("Old sequenceId received: " + requestId
+ + ", Re-sending last response to client");
+ return containerInfo.lastResponse;
+ }
+ }
+
+ TaskHeartbeatResponse tResponse = null;
+
+
+ TezTaskAttemptID taskAttemptID = request.getCurrentTaskAttemptID();
+ if (taskAttemptID != null) {
+ synchronized (containerInfo) {
+ ContainerId containerIdFromMap = attemptToContainerMap.get(new TaskAttempt(taskAttemptID));
+ if (containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
+ throw new TezException("Attempt " + taskAttemptID
+ + " is not recognized for heartbeat");
+ }
+
+ if (containerInfo.lastRequestId + 1 != requestId) {
+ throw new TezException("Container " + containerId
+ + " has invalid request id. Expected: "
+ + containerInfo.lastRequestId + 1
+ + " and actual: " + requestId);
+ }
+ }
+ TaskHeartbeatRequest tRequest = new TaskHeartbeatRequest(request.getContainerIdentifier(),
+ request.getCurrentTaskAttemptID(), request.getEvents(), request.getStartIndex(),
+ request.getMaxEvents());
+ tResponse = taskCommunicatorContext.heartbeat(tRequest);
+ }
+ TezHeartbeatResponse response;
+ if (tResponse == null) {
+ response = new TezHeartbeatResponse();
+ } else {
+ response = new TezHeartbeatResponse(tResponse.getEvents());
+ }
+ response.setLastRequestId(requestId);
+ containerInfo.lastRequestId = requestId;
+ containerInfo.lastResponse = response;
+ return response;
+ }
+
+
+ // TODO Remove this method once we move to the Protobuf RPC engine
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+ return versionID;
+ }
+
+ // TODO Remove this method once we move to the Protobuf RPC engine
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
+ int clientMethodsHash) throws IOException {
+ return ProtocolSignature.getProtocolSignature(this, protocol,
+ clientVersion, clientMethodsHash);
+ }
+ }
+
+ private ContainerTask getContainerTask(ContainerId containerId) throws IOException {
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ ContainerTask task = null;
+ if (containerInfo == null) {
+ if (taskCommunicatorContext.isKnownContainer(containerId)) {
+ LOG.info("Container with id: " + containerId
+ + " is valid, but no longer registered, and will be killed");
+ } else {
+ LOG.info("Container with id: " + containerId
+ + " is invalid and will be killed");
+ }
+ task = TASK_FOR_INVALID_JVM;
+ } else {
+ synchronized (containerInfo) {
+ if (containerInfo.taskSpec != null) {
+ if (!containerInfo.taskPulled) {
+ containerInfo.taskPulled = true;
+ task = constructContainerTask(containerInfo);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Task " + containerInfo.taskSpec.getTaskAttemptID() +
+ " already sent to container: " + containerId);
+ }
+ task = null;
+ }
+ } else {
+ task = null;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No task assigned yet for running container: " + containerId);
+ }
+ }
+ }
+ }
+ return task;
+ }
+
+ private ContainerTask constructContainerTask(ContainerInfo containerInfo) throws IOException {
+ return new ContainerTask(containerInfo.taskSpec, false,
+ convertLocalResourceMap(containerInfo.additionalLRs), containerInfo.credentials,
+ containerInfo.credentialsChanged);
+ }
+
+ private Map<String, TezLocalResource> convertLocalResourceMap(Map<String, LocalResource> ylrs)
+ throws IOException {
+ Map<String, TezLocalResource> tlrs = Maps.newHashMap();
+ if (ylrs != null) {
+ for (Map.Entry<String, LocalResource> ylrEntry : ylrs.entrySet()) {
+ TezLocalResource tlr;
+ try {
+ tlr = TezConverterUtils.convertYarnLocalResourceToTez(ylrEntry.getValue());
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ tlrs.put(ylrEntry.getKey(), tlr);
+ }
+ }
+ return tlrs;
+ }
+
+
+ // Holder for Task information, which eventually will likely be VertexImplm taskIndex, attemptIndex
+ private static class TaskAttempt {
+ // TODO TEZ-2003 Change this to work with VertexName, int id, int version
+ // TODO TEZ-2003 Avoid constructing this unit all over the place
+ private TezTaskAttemptID taskAttemptId;
+
+ TaskAttempt(TezTaskAttemptID taskAttemptId) {
+ this.taskAttemptId = taskAttemptId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof TaskAttempt)) {
+ return false;
+ }
+
+ TaskAttempt that = (TaskAttempt) o;
+
+ if (!taskAttemptId.equals(that.taskAttemptId)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return taskAttemptId.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "TaskAttempt{" + "taskAttemptId=" + taskAttemptId + '}';
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 9faf8c0..e9ba9d7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -59,6 +59,8 @@ import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskAttemptListenerImpTezDag;
+import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
@@ -86,7 +88,7 @@ public class LocalContainerLauncher extends AbstractService implements
private static final Logger LOG = LoggerFactory.getLogger(LocalContainerLauncher.class);
private final AppContext context;
- private final TaskAttemptListener taskAttemptListener;
+ private final TezTaskUmbilicalProtocol taskUmbilicalProtocol;
private final AtomicBoolean serviceStopped = new AtomicBoolean(false);
private final String workingDirectory;
private final Map<String, String> localEnv = new HashMap<String, String>();
@@ -114,7 +116,9 @@ public class LocalContainerLauncher extends AbstractService implements
String workingDirectory) throws UnknownHostException {
super(LocalContainerLauncher.class.getName());
this.context = context;
- this.taskAttemptListener = taskAttemptListener;
+ TaskAttemptListenerImpTezDag taListener = (TaskAttemptListenerImpTezDag)taskAttemptListener;
+ TezTaskCommunicatorImpl taskComm = (TezTaskCommunicatorImpl) taListener.getTaskCommunicator();
+ this.taskUmbilicalProtocol = taskComm.getUmbilical();
this.workingDirectory = workingDirectory;
AuxiliaryServiceHelper.setServiceDataIntoEnv(
ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, ByteBuffer.allocate(4).putInt(0), localEnv);
@@ -215,7 +219,7 @@ public class LocalContainerLauncher extends AbstractService implements
tezChild =
createTezChild(context.getAMConf(), event.getContainerId(), tokenIdentifier,
context.getApplicationAttemptId().getAttemptId(), context.getLocalDirs(),
- (TezTaskUmbilicalProtocol) taskAttemptListener,
+ taskUmbilicalProtocol,
TezCommonUtils.parseCredentialsBytes(event.getContainerLaunchContext().getTokens().array()));
} catch (InterruptedException e) {
handleLaunchFailed(e, event.getContainerId());
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
index a6b403d..0fc2e12 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -32,5 +33,5 @@ public interface AMContainer extends EventHandler<AMContainerEvent>{
public Container getContainer();
public List<TezTaskAttemptID> getAllTaskAttempts();
public TezTaskAttemptID getCurrentTaskAttempt();
-
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
index 682cd02..0398882 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
@@ -27,6 +27,8 @@ import org.apache.tez.runtime.api.impl.TaskSpec;
public class AMContainerEventAssignTA extends AMContainerEvent {
+ // TODO TEZ-2003. Add the task priority to this event.
+
private final TezTaskAttemptID attemptId;
// TODO Maybe have tht TAL pull the remoteTask from the TaskAttempt itself ?
private final TaskSpec remoteTaskSpec;
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 330f2b7..1acec9c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index 18286b5..0b2ea2f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.tez.dag.app.dag.DAG;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.service.AbstractService;
@@ -50,7 +51,10 @@ import org.apache.tez.client.TezApiVersionInfo;
import org.apache.tez.common.ContainerContext;
import org.apache.tez.common.ContainerTask;
import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.TaskHeartbeatRequest;
+import org.apache.tez.dag.api.TaskHeartbeatResponse;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.launcher.ContainerLauncher;
import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
@@ -72,8 +76,6 @@ import org.apache.tez.runtime.api.impl.TaskSpec;
import org.apache.tez.runtime.api.impl.TaskStatistics;
import org.apache.tez.runtime.api.impl.TezEvent;
import org.apache.tez.runtime.api.impl.EventMetaData.EventProducerConsumerType;
-import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
-import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -126,6 +128,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
Map<ContainerId, ContainerData> containers = Maps.newConcurrentMap();
ArrayBlockingQueue<Worker> workers;
TaskAttemptListenerImpTezDag taListener;
+ TezTaskCommunicatorImpl taskCommunicator;
AtomicBoolean startScheduling = new AtomicBoolean(true);
AtomicBoolean goFlag;
@@ -188,6 +191,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
@Override
public void serviceStart() throws Exception {
taListener = (TaskAttemptListenerImpTezDag) getTaskAttemptListener();
+ taskCommunicator = (TezTaskCommunicatorImpl) taListener.getTaskCommunicator();
eventHandlingThread = new Thread(this);
eventHandlingThread.start();
ExecutorService rawExecutor = Executors.newFixedThreadPool(handlerConcurrency,
@@ -327,10 +331,10 @@ public class MockDAGAppMaster extends DAGAppMaster {
}
}
- private void doHeartbeat(TezHeartbeatRequest request, ContainerData cData) throws Exception {
+ private void doHeartbeat(TaskHeartbeatRequest request, ContainerData cData) throws Exception {
long startTime = System.nanoTime();
long startCpuTime = threadMxBean.getCurrentThreadCpuTime();
- TezHeartbeatResponse response = taListener.heartbeat(request);
+ TaskHeartbeatResponse response = taListener.heartbeat(request);
if (response.shouldDie()) {
cData.remove();
} else {
@@ -381,7 +385,8 @@ public class MockDAGAppMaster extends DAGAppMaster {
try {
if (cData.taId == null) {
// if container is not assigned a task, ask for a task
- ContainerTask cTask = taListener.getTask(new ContainerContext(cData.cIdStr));
+ ContainerTask cTask =
+ taskCommunicator.getUmbilical().getTask(new ContainerContext(cData.cIdStr));
if (cTask != null) {
if (cTask.shouldDie()) {
cData.remove();
@@ -427,8 +432,9 @@ public class MockDAGAppMaster extends DAGAppMaster {
float progress = updateProgress ? cData.numUpdates/maxUpdates : 0f;
events.add(new TezEvent(new TaskStatusUpdateEvent(counters, progress, stats), new EventMetaData(
EventProducerConsumerType.SYSTEM, cData.vName, "", cData.taId)));
- TezHeartbeatRequest request = new TezHeartbeatRequest(cData.numUpdates, events,
- cData.cIdStr, cData.taId, cData.nextFromEventId, 10000);
+ TaskHeartbeatRequest request =
+ new TaskHeartbeatRequest(cData.cIdStr, cData.taId, events, cData.nextFromEventId,
+ 10000);
doHeartbeat(request, cData);
} else if (version != null && cData.taId.getId() <= version.intValue()) {
preemptContainer(cData);
@@ -438,8 +444,9 @@ public class MockDAGAppMaster extends DAGAppMaster {
List<TezEvent> events = Collections.singletonList(new TezEvent(
new TaskAttemptCompletedEvent(), new EventMetaData(
EventProducerConsumerType.SYSTEM, cData.vName, "", cData.taId)));
- TezHeartbeatRequest request = new TezHeartbeatRequest(++cData.numUpdates, events,
- cData.cIdStr, cData.taId, cData.nextFromEventId, 10000);
+ TaskHeartbeatRequest request =
+ new TaskHeartbeatRequest(cData.cIdStr, cData.taId, events, cData.nextFromEventId,
+ 10000);
doHeartbeat(request, cData);
cData.clear();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/c3c22855/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
index 1f5d9bb..8816ab5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
@@ -1,16 +1,16 @@
/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.tez.dag.app;
@@ -18,6 +18,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
@@ -28,11 +29,14 @@ import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.common.ContainerContext;
import org.apache.tez.common.ContainerTask;
-import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.rm.container.AMContainer;
import org.apache.tez.dag.app.rm.container.AMContainerMap;
@@ -55,10 +59,18 @@ public class TestTaskAttemptListenerImplTezDag {
doReturn(dag).when(appContext).getCurrentDAG();
doReturn(appAcls).when(appContext).getApplicationACLs();
doReturn(amContainerMap).when(appContext).getAllContainers();
+ NodeId nodeId = NodeId.newInstance("localhost", 0);
+ AMContainer amContainer = mock(AMContainer.class);
+ Container container = mock(Container.class);
+ doReturn(nodeId).when(container).getNodeId();
+ doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
+ doReturn(container).when(amContainer).getContainer();
TaskAttemptListenerImpTezDag taskAttemptListener =
- new TaskAttemptListenerImplForTest(appContext, mock(TaskHeartbeatHandler.class),
+ new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
mock(ContainerHeartbeatHandler.class), null);
+ TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator();
+ TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
TaskSpec taskSpec = mock(TaskSpec.class);
@@ -69,33 +81,31 @@ public class TestTaskAttemptListenerImplTezDag {
ContainerId containerId1 = createContainerId(appId, 1);
- doReturn(mock(AMContainer.class)).when(amContainerMap).get(containerId1);
ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
- containerTask = taskAttemptListener.getTask(containerContext1);
+ containerTask = tezUmbilical.getTask(containerContext1);
assertTrue(containerTask.shouldDie());
ContainerId containerId2 = createContainerId(appId, 2);
- doReturn(mock(AMContainer.class)).when(amContainerMap).get(containerId2);
ContainerContext containerContext2 = new ContainerContext(containerId2.toString());
taskAttemptListener.registerRunningContainer(containerId2);
- containerTask = taskAttemptListener.getTask(containerContext2);
+ containerTask = tezUmbilical.getTask(containerContext2);
assertNull(containerTask);
// Valid task registered
taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2);
- containerTask = taskAttemptListener.getTask(containerContext2);
+ containerTask = tezUmbilical.getTask(containerContext2);
assertFalse(containerTask.shouldDie());
assertEquals(taskSpec, containerTask.getTaskSpec());
// Task unregistered. Should respond to heartbeats
taskAttemptListener.unregisterTaskAttempt(taskAttemptId);
- containerTask = taskAttemptListener.getTask(containerContext2);
+ containerTask = tezUmbilical.getTask(containerContext2);
assertNull(containerTask);
// Container unregistered. Should send a shouldDie = true
taskAttemptListener.unregisterRunningContainer(containerId2);
- containerTask = taskAttemptListener.getTask(containerContext2);
+ containerTask = tezUmbilical.getTask(containerContext2);
assertTrue(containerTask.shouldDie());
ContainerId containerId3 = createContainerId(appId, 3);
@@ -109,7 +119,7 @@ public class TestTaskAttemptListenerImplTezDag {
AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec, null, null, false, 0);
taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3);
taskAttemptListener.unregisterRunningContainer(containerId3);
- containerTask = taskAttemptListener.getTask(containerContext3);
+ containerTask = tezUmbilical.getTask(containerContext3);
assertTrue(containerTask.shouldDie());
}
@@ -125,10 +135,18 @@ public class TestTaskAttemptListenerImplTezDag {
doReturn(dag).when(appContext).getCurrentDAG();
doReturn(appAcls).when(appContext).getApplicationACLs();
doReturn(amContainerMap).when(appContext).getAllContainers();
+ NodeId nodeId = NodeId.newInstance("localhost", 0);
+ AMContainer amContainer = mock(AMContainer.class);
+ Container container = mock(Container.class);
+ doReturn(nodeId).when(container).getNodeId();
+ doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
+ doReturn(container).when(amContainer).getContainer();
TaskAttemptListenerImpTezDag taskAttemptListener =
- new TaskAttemptListenerImplForTest(appContext, mock(TaskHeartbeatHandler.class),
+ new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
mock(ContainerHeartbeatHandler.class), null);
+ TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator();
+ TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
TaskSpec taskSpec = mock(TaskSpec.class);
@@ -139,20 +157,19 @@ public class TestTaskAttemptListenerImplTezDag {
ContainerId containerId1 = createContainerId(appId, 1);
- doReturn(mock(AMContainer.class)).when(amContainerMap).get(containerId1);
ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
taskAttemptListener.registerRunningContainer(containerId1);
- containerTask = taskAttemptListener.getTask(containerContext1);
+ containerTask = tezUmbilical.getTask(containerContext1);
assertNull(containerTask);
// Register task
taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1);
- containerTask = taskAttemptListener.getTask(containerContext1);
+ containerTask = tezUmbilical.getTask(containerContext1);
assertFalse(containerTask.shouldDie());
assertEquals(taskSpec, containerTask.getTaskSpec());
// Try pulling again - simulates re-use pull
- containerTask = taskAttemptListener.getTask(containerContext1);
+ containerTask = tezUmbilical.getTask(containerContext1);
assertNull(containerTask);
}
@@ -162,13 +179,11 @@ public class TestTaskAttemptListenerImplTezDag {
return containerId;
}
- private static class TaskAttemptListenerImplForTest extends TaskAttemptListenerImpTezDag {
+ private static class TezTaskCommunicatorImplForTest extends TezTaskCommunicatorImpl {
- public TaskAttemptListenerImplForTest(AppContext context,
- TaskHeartbeatHandler thh,
- ContainerHeartbeatHandler chh,
- JobTokenSecretManager jobTokenSecretManager) {
- super(context, thh, chh, jobTokenSecretManager);
+ public TezTaskCommunicatorImplForTest(
+ TaskCommunicatorContext taskCommunicatorContext) {
+ super(taskCommunicatorContext);
}
@Override
[25/50] [abbrv] tez git commit: TEZ-2345. Tez UI: Enable cell level
loading in all DAGs table (Sreenath Somarajapuram via pramachandran)
Posted by ss...@apache.org.
TEZ-2345. Tez UI: Enable cell level loading in all DAGs table (Sreenath Somarajapuram via pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9faa54c2
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9faa54c2
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9faa54c2
Branch: refs/heads/TEZ-2003
Commit: 9faa54c2c356cccbab7e2c2350bffc6fba7019bd
Parents: ec45c51
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Wed Apr 22 13:57:23 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Wed Apr 22 13:57:23 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../app/scripts/controllers/dags_controller.js | 199 +++++++++++--------
.../app/scripts/mixins/paginated_content.js | 83 +++-----
.../main/webapp/app/scripts/views/dropdown.js | 35 ++++
tez-ui/src/main/webapp/app/styles/colors.less | 1 +
tez-ui/src/main/webapp/app/styles/main.less | 186 +++++++++++------
tez-ui/src/main/webapp/app/styles/shared.less | 10 +-
tez-ui/src/main/webapp/app/templates/dags.hbs | 79 +++++++-
8 files changed, 387 insertions(+), 207 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/9faa54c2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 35cf312..5c91872 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2345. Tez UI: Enable cell level loading in all DAGs table
TEZ-2330. Create reconfigureVertex() API for input based initialization
TEZ-2292. Add e2e test for error reporting when vertex manager invokes
plugin APIs
http://git-wip-us.apache.org/repos/asf/tez/blob/9faa54c2/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
index f6a7013..94c6505 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
@@ -28,7 +28,6 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
// query parameters supported through url. The same named variables in this controller get
// bound automatically to the ones defined in the route.
queryParams: {
- fromID: true,
status_filter: 'status',
user_filter: 'user',
appId_filter: 'appid',
@@ -38,14 +37,38 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
fromID: null,
status_filter: null,
-
user_filter: null,
-
appId_filter: null,
-
dagName_filter: null,
- fields: 'events,primaryfilters,otherinfo',
+ boundFilterValues: Em.Object.create({
+ status: null
+ }),
+ visibleFilters: null,
+
+ init: function () {
+ this._super();
+ this._filterVisiblilityObserver();
+ },
+
+ _paramObserver: function () {
+ this.set('boundFilterValues', Em.Object.create({
+ status: this.get('status_filter'),
+ user: this.get('user_filter'),
+ appId: this.get('appId_filter'),
+ dagName: this.get('dagName_filter')
+ }));
+ }.observes('status_filter', 'user_filter', 'appId_filter', 'dagName_filter'),
+
+ _filterVisiblilityObserver: function () {
+ var visibleFilters = Em.Object.create();
+ this.get('columns').forEach(function (column) {
+ if(column.get('enableFilter')) {
+ visibleFilters.set(column.get('id'), true);
+ }
+ });
+ this.set('visibleFilters', visibleFilters);
+ }.observes('columns'),
loadData: function() {
var filters = {
@@ -69,7 +92,6 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
var that = this,
store = this.get('store'),
childEntityType = this.get('childEntityType'),
- fetcher,
record;
var defaultErrMsg = 'Error while loading dag info.';
@@ -78,16 +100,22 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
store.unloadAll('dagProgress');
store.findQuery(childEntityType, this.getFilterProperties()).then(function(entities){
- var loaders = [];
+ that.set('entities', entities);
+ that.set('loading', false);
+
entities.forEach(function (dag) {
var appId = dag.get('applicationId');
if(appId) {
- // Pivot attempt selection logic
record = store.getById('appDetail', appId);
if(record && !App.Helpers.misc.isStatusInUnsuccessful(record.get('appState'))) {
store.unloadRecord(record);
}
- fetcher = store.find('appDetail', appId).then(function (app) {
+ }
+ });
+ entities.forEach(function (dag) {
+ var appId = dag.get('applicationId');
+ if(appId) {
+ store.find('appDetail', appId).then(function (app) {
dag.set('appDetail', app);
if (dag.get('status') === 'RUNNING') {
dag.set('status', App.Helpers.misc.getRealStatus(
@@ -95,17 +123,15 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
app.get('appState'),
app.get('finalAppStatus')
));
- App.Helpers.misc.removeRecord(store, 'tezApp', 'tez_' + appId);
}
- return store.find('tezApp', 'tez_' + appId).then(function (app) {
- dag.set('tezApp', app);
- });
+ })
+ .catch(function(error) {
+ Em.Logger.error('Failed to fetch appDetail' + error);
});
- loaders.push(fetcher);
- //Load tezApp details
+
if (dag.get('status') === 'RUNNING') {
App.Helpers.misc.removeRecord(store, 'dagProgress', dag.get('id'));
- amInfoFetcher = store.find('dagProgress', dag.get('id'), {
+ store.find('dagProgress', dag.get('id'), {
appId: dag.get('applicationId'),
dagIdx: dag.get('idx')
})
@@ -115,32 +141,29 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
.catch(function(error) {
Em.Logger.error('Failed to fetch dagProgress' + error);
});
- loaders.push(amInfoFetcher);
}
}
});
- Em.RSVP.allSettled(loaders).then(function(){
- that.set('entities', entities);
- that.set('loading', false);
- });
}).catch(function(error){
Em.Logger.error(error);
var err = App.Helpers.misc.formatError(error, defaultErrMsg);
var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
App.Helpers.ErrorBar.getInstance().show(msg, err.details);
});
- },
+ }.observes('fields'),
actions : {
- filterUpdated: function(filterID, value) {
- // any validations required goes here.
- if (!!value) {
- this.set(filterID, value);
- } else {
- this.set(filterID, null);
- }
+ filterUpdated: function() {
+ Em.run.later();
+ var filterValues = this.get('boundFilterValues');
+ this.setProperties({
+ status_filter: filterValues.get('status') || null,
+ user_filter: filterValues.get('user') || null,
+ appId_filter: filterValues.get('appId') || null,
+ dagName_filter: filterValues.get('dagName') || null,
+ });
this.loadData();
- },
+ }
},
/*
@@ -148,19 +171,32 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
* @return Array of column configs
*/
defaultColumnConfigs: function () {
+ var store = this.get('store');
+
+ function onProgressChange() {
+ var progress = this.get('dag.progress'),
+ pct;
+ if (Ember.typeOf(progress) === 'number') {
+ pct = App.Helpers.number.fractionToPercentage(progress);
+ this.set('progress', pct);
+ }
+ }
+
+ function onStatusChange() {
+ this.set('status', this.get('dag.status'));
+ }
+
return [
{
id: 'dagName',
headerCellName: 'Dag Name',
- filterID: 'dagName_filter',
- tableCellViewClass: Em.Table.TableCell.extend({
- template: Em.Handlebars.compile(
- "{{#link-to 'dag.index' view.cellContent.id class='ember-table-content'}}{{view.cellContent.name}}{{/link-to}}")
- }),
+ templateName: 'components/basic-table/linked-cell',
+ enableFilter: true,
getCellContent: function(row) {
return {
- id: row.get('id'),
- name: row.get('name')
+ linkTo: 'dag.index',
+ entityId: row.get('id'),
+ displayText: row.get('name')
};
}
},
@@ -172,38 +208,34 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
{
id: 'user',
headerCellName: 'Submitter',
- filterID: 'user_filter',
- contentPath: 'user'
+ contentPath: 'user',
+ enableFilter: true
},
{
id: 'status',
headerCellName: 'Status',
- filterID: 'status_filter',
- filterType: 'dropdown',
- dropdownValues: App.Helpers.misc.dagStatusUIOptions,
- tableCellViewClass: Em.Table.TableCell.extend({
- template: Em.Handlebars.compile(
- '<span class="ember-table-content"> \
- <i {{bind-attr class=":task-status view.cellContent.statusIcon"}}></i>\
- {{view.cellContent.status}}\
- {{#if view.cellContent.progress}} {{bs-badge content=view.cellContent.progress}}{{/if}}</span>')
- }),
+ templateName: 'components/basic-table/status-cell',
+ enableFilter: true,
getCellContent: function(row) {
- var pct;
- if (Ember.typeOf(row.get('progress')) === 'number') {
- pct = App.Helpers.number.fractionToPercentage(row.get('progress'));
+ var status = row.get('status'),
+ content = Ember.Object.create({
+ dag: row,
+ status: status,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(status)
+ });
+
+ if(status == 'RUNNING') {
+ row.addObserver('progress', content, onProgressChange);
+ row.addObserver('status', content, onStatusChange);
}
- var dagStatus = row.get('status');
- return {
- status: dagStatus,
- statusIcon: App.Helpers.misc.getStatusClassForEntity(dagStatus),
- progress: pct
- };
+
+ return content;
}
},
{
id: 'startTime',
headerCellName: 'Start Time',
+ contentPath: 'startTime',
getCellContent: function(row) {
return App.Helpers.date.dateFormat(row.get('startTime'));
}
@@ -219,37 +251,44 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
id: 'duration',
headerCellName: 'Duration',
getCellContent: function(row) {
- var st = row.get('startTime');
- var et = row.get('endTime');
- if (st && et) {
- return App.Helpers.date.durationSummary(st, et);
- }
+ return App.Helpers.date.timingFormat(row.get('duration'), 1);
}
},
{
id: 'appId',
headerCellName: 'Application ID',
- filterID: 'appId_filter',
- tableCellViewClass: Em.Table.TableCell.extend({
- template: Em.Handlebars.compile(
- "{{#if view.cellContent.enableLink}}\
- {{#link-to 'tez-app' view.cellContent.appId class='ember-table-content'}}{{view.cellContent.appId}}{{/link-to}}\
- {{else}}\
- <span class='ember-table-content'>{{view.cellContent.appId}}</span>\
- {{/if}}")
- }),
+ templateName: 'components/basic-table/linked-cell',
+ enableFilter: true,
getCellContent: function(row) {
- return {
- enableLink: row.get('tezApp') && row.get('appDetail'),
- appId: row.get('applicationId')
+ var appId = row.get('applicationId');
+ if(appId) {
+ return Em.RSVP.allSettled([
+ store.find('appDetail', appId),
+ store.find('tezApp', 'tez_' + appId)
+ ]).then(function (response) {
+ var content = {
+ displayText: row.get('applicationId'),
+ entityId: row.get('applicationId')
+ };
+ if(response.get('0.value') && response.get('1.value')) {
+ content.linkTo = 'tez-app';
+ }
+ return content;
+ });
}
}
},
{
id: 'queue',
headerCellName: 'Queue',
+ templateName: 'components/basic-table/bounded-basic-cell',
getCellContent: function(row) {
- return row.get('appDetail.queue') || 'Not Available';
+ var appId = row.get('applicationId');
+ if(appId) {
+ return store.find('appDetail', appId).then(function (app) {
+ return app.get('queue');
+ });
+ }
}
}
];
@@ -266,12 +305,4 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
);
}.property(),
- columns: function() {
- var visibleColumnConfigs = this.get('columnConfigs').filter(function (column) {
- return this.visibleColumnIds[column.id];
- }, this);
-
- return App.Helpers.misc.createColumnsFromConfigs(visibleColumnConfigs);
- }.property('visibleColumnIds'),
-
});
http://git-wip-us.apache.org/repos/asf/tez/blob/9faa54c2/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js b/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
index 5e158e1..13e27cc 100644
--- a/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
+++ b/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
@@ -19,13 +19,13 @@
App.PaginatedContentMixin = Em.Mixin.create({
// paging related values. These are bound automatically to the values in url. via the queryParams
// defined in the route.
- count: 10,
+ rowCount: 10,
page: 1,
fromID: null,
// The dropdown contents for number of items to show.
- countOptions: [5, 10, 25, 50, 100],
+ rowCountOptions: [5, 10, 25, 50, 100],
isRefreshable: true,
@@ -33,14 +33,10 @@ App.PaginatedContentMixin = Em.Mixin.create({
* store the first dag id on a page so that we can navigate back and store the last one
* (not shown on page to get the id where next page starts)
*/
- navIDs: {
- prevIDs: [],
- currentID: undefined,
- nextID: undefined
- },
+ navIDs: [],
queryParams: {
- count: true,
+ rowCount: true,
},
entities: [],
@@ -50,7 +46,11 @@ App.PaginatedContentMixin = Em.Mixin.create({
load: function() {
this.resetNavigation();
this.loadEntities();
- }.observes('count'),
+ }.observes('rowCount'),
+
+ lastPage: function () {
+ return this.get('navIDs.length') + 1;
+ }.property('navIDs.length'),
sortedContent: function() {
// convert to a ArrayController. we do not sort at this point as the data is
@@ -59,7 +59,7 @@ App.PaginatedContentMixin = Em.Mixin.create({
model: this.get('entities')
});
this.updatePagination(sorted.toArray());
- return sorted.slice(0, this.count);
+ return sorted.slice(0, this.rowCount);
}.property('entities', 'numEntities'),
updateLoading: function () {
@@ -104,65 +104,36 @@ App.PaginatedContentMixin = Em.Mixin.create({
},
resetNavigation: function() {
- this.set('navIDs.prevIDs', []);
- this.set('navIDs.currentID', '');
- this.set('navIDs.nextID', '');
+ this.set('navIDs', []);
this.set('fromID', null);
this.set('page', 1);
},
updatePagination: function(dataArray) {
- if (!!dataArray && dataArray.get('length') > 0) {
- this.set('navIDs.currentID', dataArray.objectAt(0).get('id'));
- var nextID = undefined;
- if (dataArray.get('length') > this.count) {
- // save the last id, so that we can use that as firt id on next page.
- nextID = dataArray.objectAt(this.count).get('id');
+ var nextFromId = null,
+ navIDs = this.get('navIDs'),
+ rowCount = this.get('rowCount');
+
+ if(dataArray && dataArray.length == rowCount + 1) {
+ nextFromId = dataArray.objectAt(rowCount).get('id');
+ if (navIDs.indexOf(nextFromId) == -1 &&
+ this.get('page') >= navIDs.get('length')) {
+ navIDs.pushObject(nextFromId);
}
- this.set('navIDs.nextID', nextID);
}
},
- hasPrev: function() {
- return this.navIDs.prevIDs.length > 0;
- }.property('navIDs.prevIDs.[]', 'navIDs.prevIDs.length', 'fromID', 'page'),
-
- hasNext: function() {
- return !!this.navIDs.nextID;
- }.property('navIDs.nextID'),
-
actions:{
- // go to previous page
- navigatePrev: function () {
- var prevPageId = this.navIDs.prevIDs.popObject();
- this.set('fromID', prevPageId);
- this.set('loading', true);
- this.set('page', this.get('page') - 1);
- this.loadEntities();
- },
-
refresh: function () {
this.load();
},
- // goto first page.
- navigateFirst: function() {
- var firstPageId = this.navIDs.prevIDs[0];
- this.set('navIDs.prevIDs', []);
- this.set('fromID', firstPageId);
+ changePage: function (pageNum) {
+ this.set('fromID', this.get('navIDs.' + (pageNum - 2)) || null);
this.set('loading', true);
- this.set('page', 1);
+ this.set('page', pageNum);
this.loadEntities();
- },
-
- // go to next page
- navigateNext: function () {
- this.navIDs.prevIDs.pushObject(this.navIDs.currentID);
- this.set('fromID', this.get('navIDs.nextID'));
- this.set('loading', true);
- this.set('page', this.get('page') + 1);
- this.loadEntities();
- },
+ }
},
_concatFilters: function(obj) {
@@ -177,7 +148,7 @@ App.PaginatedContentMixin = Em.Mixin.create({
getFilterProperties: function() {
var params = {
- limit: this.count + 1
+ limit: this.rowCount + 1
};
var f = this._paginationFilters;
@@ -213,8 +184,8 @@ App.PaginatedContentMixin = Em.Mixin.create({
params['secondaryFilter'] = secondary;
}
- if (!Em.empty(this.fromID)) {
- params['fromId'] = this.fromID;
+ if (!Em.empty(this.get('fromID'))) {
+ params['fromId'] = this.get('fromID');
}
return params;
http://git-wip-us.apache.org/repos/asf/tez/blob/9faa54c2/tez-ui/src/main/webapp/app/scripts/views/dropdown.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/views/dropdown.js b/tez-ui/src/main/webapp/app/scripts/views/dropdown.js
new file mode 100644
index 0000000..be97507
--- /dev/null
+++ b/tez-ui/src/main/webapp/app/scripts/views/dropdown.js
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+App.Dropdown = Em.Select.extend({
+ currentValue: null,
+ init: function () {
+ this._super();
+ this.set('currentValue', this.get('value'));
+ },
+ change: function() {
+ var value = this.get('value'),
+ target = this.get('target') || this.get('context');
+
+ if(target && value != this.get('currentValue')) {
+ Em.run.later(target.send.bind(target, this.get('action'), value), 100);
+ this.set('currentValue', value);
+ }
+ return true;
+ }
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/9faa54c2/tez-ui/src/main/webapp/app/styles/colors.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/colors.less b/tez-ui/src/main/webapp/app/styles/colors.less
index 9bf4b61..aa0d96a 100644
--- a/tez-ui/src/main/webapp/app/styles/colors.less
+++ b/tez-ui/src/main/webapp/app/styles/colors.less
@@ -32,6 +32,7 @@
@text-color: #666666;
@text-red: red;
+@text-light: #BBBBBB;
@top-nav-bg-color-from: #d5d5d5;
@top-nav-bg-color-to: #f0f0f0;
http://git-wip-us.apache.org/repos/asf/tez/blob/9faa54c2/tez-ui/src/main/webapp/app/styles/main.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/main.less b/tez-ui/src/main/webapp/app/styles/main.less
index 4687ac4..8204b19 100644
--- a/tez-ui/src/main/webapp/app/styles/main.less
+++ b/tez-ui/src/main/webapp/app/styles/main.less
@@ -56,16 +56,6 @@ body, html {
}
}
- .table-control {
- .align-children-left {
- vertical-align: top;
- padding-top: 4px;
- }
- .align-children-right {
- margin-top: -8px;
- }
- }
-
.load-component {
margin-bottom: 5px;
@@ -392,6 +382,9 @@ body, html {
.task-status {
.fa;
.fa-lg;
+ .absolute;
+
+ top: 7px;
&.success {
.fa-icon(check-circle);
@@ -424,6 +417,9 @@ body, html {
color: @unknown-color;
}
}
+.status-msg {
+ margin-left: 20px;
+}
.fa-action {
.fa;
@@ -531,6 +527,113 @@ div.indent {
color: black;
}
+.row-select {
+ margin-left: 5px;
+
+ display: inline-block;
+ text-align: center;
+
+ :first-child {
+ margin-bottom: -3px;
+ font-size: .85em;
+ }
+}
+
+.dag-header {
+ .margin-small-horizontal;
+
+ .align-children-left {
+ width: 65%;
+ .inline-block;
+ }
+ .align-children-right {
+ width: 35%;
+ .inline-block;
+ }
+
+ margin-bottom: -5px;
+ margin-top: 10px;
+
+ .pagination-view {
+ .page-list {
+ margin-bottom: 2px;
+ }
+ }
+
+ .filter-elements {
+ .inline-block;
+
+ margin-left: 5px;
+ display: inline-block;
+
+ input[type=text], select {
+ width: 110px;
+ }
+ input, select {
+ height: 20px;
+ border-radius: 5px;
+ border: 1px solid @border-color;
+ }
+
+ .load-counter-check {
+ margin-left: 10px;
+ vertical-align: sub;
+ }
+
+ :first-child {
+ margin-bottom: -3px;
+ font-size: .85em;
+
+ padding: 2px;
+ }
+ }
+}
+
+.pagination-view {
+ .inline-block;
+
+ .page-list {
+ .inline-block;
+ .align-top;
+
+ overflow: hidden;
+
+ border: 1px solid @border-color;
+ border-radius: 5px;
+
+ padding: 0px;
+
+ font-size: 0px;
+
+ li {
+ .inline-block;
+
+ padding: 6px 10px;
+ font-size: 14px;
+
+ border-left: 1px solid @border-color;
+
+ pointer-events: none;
+
+ &.clickable {
+ pointer-events: auto;
+ &:hover {
+ background-color: @bg-grey;
+ cursor: pointer;
+ }
+ }
+ }
+
+ .total-page-count {
+ font-size: .8em;
+ }
+
+ :first-child {
+ border-left: none;
+ }
+ }
+}
+
.table-container {
.use-gpu;
@@ -599,61 +702,7 @@ div.indent {
.fa-icon(spinner);
.fa-spin;
.fa-fw;
- }
-
- .pagination-view {
- .inline-block;
-
- .page-list {
- .inline-block;
- .align-top;
-
- border: 1px solid @border-color;
- border-radius: 5px;
-
- padding: 0px;
-
- font-size: 0px;
-
- li {
- .inline-block;
-
- padding: 6px 10px;
- font-size: 14px;
-
- border-left: 1px solid @border-color;
-
- pointer-events: none;
-
- &.clickable {
- pointer-events: auto;
- &:hover {
- background-color: @bg-grey;
- cursor: pointer;
- }
- }
- }
-
- .total-page-count {
- font-size: .8em;
- }
-
- :first-child {
- border-left: none;
- }
- }
-
- .row-select {
- margin-left: 5px;
-
- display: inline-block;
- text-align: center;
-
- :first-child {
- margin-bottom: -3px;
- font-size: .85em;
- }
- }
+ .absolute;
}
.table-body-container{
@@ -689,6 +738,13 @@ div.indent {
text-overflow: ellipsis;
white-space: nowrap;
height: 18px;
+
+ .message {
+ color: @text-light;
+ .waiting {
+ color: @text-color;
+ }
+ }
}
.sort-icon {
http://git-wip-us.apache.org/repos/asf/tez/blob/9faa54c2/tez-ui/src/main/webapp/app/styles/shared.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/shared.less b/tez-ui/src/main/webapp/app/styles/shared.less
index cebd837..7cece16 100644
--- a/tez-ui/src/main/webapp/app/styles/shared.less
+++ b/tez-ui/src/main/webapp/app/styles/shared.less
@@ -37,6 +37,10 @@
visibility: hidden !important;
}
+.no-margin {
+ margin: 0px !important;
+}
+
.no-pointer {
pointer-events: none;
}
@@ -45,10 +49,14 @@
white-space: nowrap;
}
-.align-top{
+.align-top {
vertical-align: top;
}
+.align-super {
+ vertical-align: super;
+}
+
.inline-block {
display: inline-block;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/9faa54c2/tez-ui/src/main/webapp/app/templates/dags.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/dags.hbs b/tez-ui/src/main/webapp/app/templates/dags.hbs
index d0e83c4..727e28a 100644
--- a/tez-ui/src/main/webapp/app/templates/dags.hbs
+++ b/tez-ui/src/main/webapp/app/templates/dags.hbs
@@ -21,5 +21,82 @@
</ul>
<div class='margin-small-vertical'>
- {{partial 'common/table-with-spinner'}}
+ {{#unless loading}}
+ {{load-time-component
+ isRefreshable=isRefreshable
+ time=sortedContent.0.timeStamp
+ refresh='refresh'
+ }}
+ <div class='dag-header'>
+ <div class="align-children-left">
+ {{#if visibleFilters.dagName}}
+ <div class='filter-elements'>
+ <div>Dag Name</div>
+ {{input
+ action="filterUpdated"
+ value=boundFilterValues.dagName
+ placeholder="Search..."
+ }}
+ </div>
+ {{/if}}
+ {{#if visibleFilters.user}}
+ <div class='filter-elements'>
+ <div>Submitter</div>
+ {{input
+ action="filterUpdated"
+ value=boundFilterValues.user
+ placeholder="Search..."
+ }}
+ </div>
+ {{/if}}
+ {{#if visibleFilters.status}}
+ <div class='filter-elements'>
+ <div>Status</div>
+ {{view App.Dropdown
+ optionValuePath='content.id'
+ optionLabelPath='content.label'
+ classNames='inline-display'
+ action='filterUpdated'
+ content=App.Helpers.misc.dagStatusUIOptions
+ value=boundFilterValues.status
+ }}
+ </div>
+ {{/if}}
+ {{#if visibleFilters.appId}}
+ <div class='filter-elements'>
+ <div>Application ID</div>
+ {{input
+ action="filterUpdated"
+ value=boundFilterValues.appId
+ placeholder="Search..."
+ }}
+ </div>
+ {{/if}}
+ </div><div class="align-children-right">
+ {{view App.BasicTableComponent.PaginationView
+ pageNum=page
+ totalPages=lastPage
+ hideLast=true
+ }}
+ <i {{bind-attr class=':fa-action :fa-cog :left-divider'}} {{action 'selectColumns'}}></i>
+ </div>
+ </div>
+ {{basic-table-component
+ columns=columns
+ rows=sortedContent
+
+ rowCountBinding='rowCount'
+
+ statusMessage=statusMessage
+ }}
+ {{#unless sortedContent.length}}
+ <h1 class="no-margin">No records available!</h1>
+ {{/unless}}
+ {{else}}
+ {{partial 'partials/loading-spinner'}}
+ <div class="text-align-center">
+ {{statusMessage}}
+ </div>
+ {{/unless}}
+
</div>
[29/50] [abbrv] tez git commit: TEZ-2352. Move getTaskStatistics into
the RuntimeTask class. (sseth)
Posted by ss...@apache.org.
TEZ-2352. Move getTaskStatistics into the RuntimeTask class. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ec7b481b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ec7b481b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ec7b481b
Branch: refs/heads/TEZ-2003
Commit: ec7b481b691504677b00e9461911917e8cd2f791
Parents: 67609af
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Apr 23 12:25:44 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 12:25:44 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/runtime/LogicalIOProcessorRuntimeTask.java | 8 --------
.../src/main/java/org/apache/tez/runtime/RuntimeTask.java | 7 +++++++
.../main/java/org/apache/tez/runtime/task/TaskReporter.java | 9 ++++-----
4 files changed, 12 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7b481b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1c41714..43767d2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2352. Move getTaskStatistics into the RuntimeTask class.
TEZ-2357. Tez UI: misc.js.orig is committed by accident
TEZ-2261. Should add diagnostics in DAGAppMaster when recovery error happens
TEZ-2340. TestRecoveryParser fails
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7b481b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
index 0b048da..80c2717 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
@@ -73,7 +73,6 @@ import org.apache.tez.runtime.api.impl.GroupInputSpec;
import org.apache.tez.runtime.api.impl.InputSpec;
import org.apache.tez.runtime.api.impl.OutputSpec;
import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.api.impl.TaskStatistics;
import org.apache.tez.runtime.api.impl.TezEvent;
import org.apache.tez.runtime.api.impl.TezInputContextImpl;
import org.apache.tez.runtime.api.impl.TezMergedInputContextImpl;
@@ -118,8 +117,6 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask {
private final LinkedHashMap<String, LogicalInput> runInputMap;
private final LinkedHashMap<String, LogicalOutput> runOutputMap;
- private final TaskStatistics statistics;
-
private final Map<String, ByteBuffer> serviceConsumerMetadata;
private final Map<String, String> envMap;
@@ -182,7 +179,6 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask {
this.objectRegistry = objectRegistry;
this.ExecutionContext = ExecutionContext;
this.memAvailable = memAvailable;
- this.statistics = new TaskStatistics();
}
/**
@@ -325,10 +321,6 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask {
startRouterThread();
}
-
- public TaskStatistics getTaskStatistics() {
- return statistics;
- }
public void run() throws Exception {
Preconditions.checkState(this.state.get() == State.INITED,
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7b481b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java
index 4777b71..745b10b 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.runtime.api.impl.TaskStatistics;
import org.apache.tez.runtime.api.impl.TezEvent;
import org.apache.tez.runtime.api.impl.TezUmbilical;
import org.apache.tez.runtime.metrics.TaskCounterUpdater;
@@ -49,6 +50,7 @@ public abstract class RuntimeTask {
protected final AtomicInteger eventCounter;
private final AtomicBoolean taskDone;
private final TaskCounterUpdater counterUpdater;
+ private final TaskStatistics statistics;
protected RuntimeTask(TaskSpec taskSpec, Configuration tezConf,
TezUmbilical tezUmbilical, String pid) {
@@ -59,6 +61,7 @@ public abstract class RuntimeTask {
this.eventCounter = new AtomicInteger(0);
this.progress = 0.0f;
this.taskDone = new AtomicBoolean(false);
+ this.statistics = new TaskStatistics();
this.counterUpdater = new TaskCounterUpdater(tezCounters, tezConf, pid);
}
@@ -109,6 +112,10 @@ public abstract class RuntimeTask {
return fullCounters;
}
+ public TaskStatistics getTaskStatistics() {
+ return statistics;
+ }
+
public TezTaskAttemptID getTaskAttemptID() {
return taskSpec.getTaskAttemptID();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7b481b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java
index 48be8bd..b9e7217 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java
@@ -34,10 +34,9 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
-import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.LogicalIOProcessorRuntimeTask;
+import org.apache.tez.runtime.RuntimeTask;
import org.apache.tez.runtime.api.events.TaskAttemptCompletedEvent;
import org.apache.tez.runtime.api.events.TaskAttemptFailedEvent;
import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent;
@@ -96,7 +95,7 @@ public class TaskReporter {
/**
* Register a task to be tracked. Heartbeats will be sent out for this task to fetch events, etc.
*/
- public synchronized void registerTask(LogicalIOProcessorRuntimeTask task,
+ public synchronized void registerTask(RuntimeTask task,
ErrorReporter errorReporter) {
currentCallable = new HeartbeatCallable(task, umbilical, pollInterval, sendCounterInterval,
maxEventsToGet, requestCounter, containerIdStr);
@@ -123,7 +122,7 @@ public class TaskReporter {
private static final int LOG_COUNTER_START_INTERVAL = 5000; // 5 seconds
private static final float LOG_COUNTER_BACKOFF = 1.3f;
- private final LogicalIOProcessorRuntimeTask task;
+ private final RuntimeTask task;
private EventMetaData updateEventMetadata;
private final TezTaskUmbilicalProtocol umbilical;
@@ -151,7 +150,7 @@ public class TaskReporter {
*/
private int prevCounterSendHeartbeatNum = 0;
- public HeartbeatCallable(LogicalIOProcessorRuntimeTask task,
+ public HeartbeatCallable(RuntimeTask task,
TezTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval,
int maxEventsToGet, AtomicLong requestCounter, String containerIdStr) {
[46/50] [abbrv] tez git commit: TEZ-2139. Update version to
0.7.0-TEZ-2003-SNAPSHOT. (sseth)
Posted by ss...@apache.org.
TEZ-2139. Update version to 0.7.0-TEZ-2003-SNAPSHOT. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/e8f804ff
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/e8f804ff
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/e8f804ff
Branch: refs/heads/TEZ-2003
Commit: e8f804ff2d98b599c2fbc9fc7bc4eeb4be4a6f17
Parents: d78ffbf
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Feb 23 21:59:39 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:47:33 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
docs/pom.xml | 2 +-
pom.xml | 2 +-
tez-api/pom.xml | 2 +-
tez-common/pom.xml | 2 +-
tez-dag/pom.xml | 2 +-
tez-dist/pom.xml | 2 +-
tez-examples/pom.xml | 2 +-
tez-ext-service-tests/pom.xml | 2 +-
tez-mapreduce/pom.xml | 2 +-
tez-plugins/pom.xml | 2 +-
tez-plugins/tez-mbeans-resource-calculator/pom.xml | 2 +-
tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml | 2 +-
tez-plugins/tez-yarn-timeline-history/pom.xml | 2 +-
tez-runtime-internals/pom.xml | 2 +-
tez-runtime-library/pom.xml | 2 +-
tez-tests/pom.xml | 2 +-
tez-ui/pom.xml | 2 +-
18 files changed, 18 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 4377f57..5c5fd8e 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -8,5 +8,6 @@ ALL CHANGES:
TEZ-2125. Create a task communicator for local mode. Allow tasks to run in the AM.
TEZ-2131. Add additional tests for tasks running in the AM.
TEZ-2138. Fix minor bugs in adding default scheduler, getting launchers.
+ TEZ-2139. Update tez version to 0.7.0-TEZ-2003-SNAPSHOT.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index 91e827e..a55bd95 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-docs</artifactId>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 50834d3..1125ac8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
<packaging>pom</packaging>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
<name>tez</name>
<licenses>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-api/pom.xml b/tez-api/pom.xml
index e726135..5ee0a8c 100644
--- a/tez-api/pom.xml
+++ b/tez-api/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-api</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-common/pom.xml
----------------------------------------------------------------------
diff --git a/tez-common/pom.xml b/tez-common/pom.xml
index 679d1c8..55909c5 100644
--- a/tez-common/pom.xml
+++ b/tez-common/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-common</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-dag/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index 97de918..f171de6 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<properties>
<tez.component>tez-dag</tez.component>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-dist/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml
index cfc1d32..ffead81 100644
--- a/tez-dist/pom.xml
+++ b/tez-dist/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-examples/pom.xml
----------------------------------------------------------------------
diff --git a/tez-examples/pom.xml b/tez-examples/pom.xml
index f4924fb..276b60d 100644
--- a/tez-examples/pom.xml
+++ b/tez-examples/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-ext-service-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml
index 37f68b1..d64dc58 100644
--- a/tez-ext-service-tests/pom.xml
+++ b/tez-ext-service-tests/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tez</artifactId>
<groupId>org.apache.tez</groupId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<!-- TODO TEZ-2003 Merge this into the tez-tests module -->
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce/pom.xml b/tez-mapreduce/pom.xml
index f2baad0..cfd1390 100644
--- a/tez-mapreduce/pom.xml
+++ b/tez-mapreduce/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-mapreduce</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/tez-plugins/pom.xml b/tez-plugins/pom.xml
index 1080f2f..944f782 100644
--- a/tez-plugins/pom.xml
+++ b/tez-plugins/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-plugins</artifactId>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-plugins/tez-mbeans-resource-calculator/pom.xml
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-mbeans-resource-calculator/pom.xml b/tez-plugins/tez-mbeans-resource-calculator/pom.xml
index 7428029..692d6ce 100644
--- a/tez-plugins/tez-mbeans-resource-calculator/pom.xml
+++ b/tez-plugins/tez-mbeans-resource-calculator/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez-plugins</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-mbeans-resource-calculator</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml b/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml
index 6ca51b4..c8ce585 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml
+++ b/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez-plugins</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-yarn-timeline-history-with-acls</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-plugins/tez-yarn-timeline-history/pom.xml
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/pom.xml b/tez-plugins/tez-yarn-timeline-history/pom.xml
index 2978e88..04a054d 100644
--- a/tez-plugins/tez-yarn-timeline-history/pom.xml
+++ b/tez-plugins/tez-yarn-timeline-history/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez-plugins</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-yarn-timeline-history</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-runtime-internals/pom.xml
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/pom.xml b/tez-runtime-internals/pom.xml
index ad34c2d..76f7263 100644
--- a/tez-runtime-internals/pom.xml
+++ b/tez-runtime-internals/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-runtime-internals</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-runtime-library/pom.xml
----------------------------------------------------------------------
diff --git a/tez-runtime-library/pom.xml b/tez-runtime-library/pom.xml
index 0edf169..268bb90 100644
--- a/tez-runtime-library/pom.xml
+++ b/tez-runtime-library/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-runtime-library</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tez-tests/pom.xml b/tez-tests/pom.xml
index 1b3378c..f7c41ab 100644
--- a/tez-tests/pom.xml
+++ b/tez-tests/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-tests</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/e8f804ff/tez-ui/pom.xml
----------------------------------------------------------------------
diff --git a/tez-ui/pom.xml b/tez-ui/pom.xml
index ca4d7ab..df441f7 100644
--- a/tez-ui/pom.xml
+++ b/tez-ui/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tez</groupId>
<artifactId>tez</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>0.7.0-TEZ-2003-SNAPSHOT</version>
</parent>
<artifactId>tez-ui</artifactId>
<packaging>war</packaging>
[14/50] [abbrv] tez git commit: TEZ-1969. Stop the DAGAppMaster when
a local mode client is stopped (pramachandran)
Posted by ss...@apache.org.
TEZ-1969. Stop the DAGAppMaster when a local mode client is stopped (pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/3e6fc355
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/3e6fc355
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/3e6fc355
Branch: refs/heads/TEZ-2003
Commit: 3e6fc355c618b77531f7d29b6a1c9cbc3c39456a
Parents: e196868
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Fri Apr 17 07:54:43 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Fri Apr 17 07:54:43 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../src/main/java/org/apache/tez/client/LocalClient.java | 9 +++++++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/3e6fc355/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ce9bfe8..1a7609f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-1969. Stop the DAGAppMaster when a local mode client is stopped
TEZ-714. OutputCommitters should not run in the main AM dispatcher thread
TEZ-2323. Fix TestOrderedWordcount to use MR memory configs.
TEZ-1482. Fix memory issues for Local Mode running concurrent tasks
http://git-wip-us.apache.org/repos/asf/tez/blob/3e6fc355/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
index b5889a9..6eb58bd 100644
--- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
+++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
@@ -106,6 +106,14 @@ public class LocalClient extends FrameworkClient {
public void close() throws IOException {
// LocalClients are shared between TezClient and DAGClients, which can cause stop / start / close
// to be invoked multiple times. If modifying these methods - this should be factored in.
+
+ // Multiple DAGClient's can reuse the LocalClient (for ex session). However there is only a
+ // single instance of LocalClient for a TezClient, and dagAppMaster can be cleaned up when
+ // the TezClient is stopped, in order not to leak.
+ if (dagAppMaster != null) {
+ dagAppMaster.stop();
+ dagAppMaster = null;
+ }
}
@Override
@@ -309,6 +317,7 @@ public class LocalClient extends FrameworkClient {
LOG.error("Error starting DAGAppMaster", t);
if (dagAppMaster != null) {
dagAppMaster.stop();
+ dagAppMaster = null;
}
amFailException = t;
}
[48/50] [abbrv] tez git commit: TEZ-2302. Allow TaskCommunicators to
subscribe for Vertex updates. (sseth)
Posted by ss...@apache.org.
TEZ-2302. Allow TaskCommunicators to subscribe for Vertex updates. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/74c05f61
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/74c05f61
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/74c05f61
Branch: refs/heads/TEZ-2003
Commit: 74c05f61755b4359f5cc4c8ce042f91820a422ec
Parents: 7eca3b7
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Apr 9 13:33:48 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:48:02 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../apache/tez/dag/api/TaskCommunicator.java | 20 +++
.../tez/dag/api/TaskCommunicatorContext.java | 14 ++-
.../dag/app/TaskAttemptListenerImpTezDag.java | 52 +++-----
.../dag/app/TaskCommunicatorContextImpl.java | 124 +++++++++++++++++++
.../tez/dag/app/TezTaskCommunicatorImpl.java | 6 +
.../java/org/apache/tez/dag/app/dag/DAG.java | 2 +
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 5 +
8 files changed, 188 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/74c05f61/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 9d6b220..ca5225e 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -15,5 +15,6 @@ ALL CHANGES:
TEZ-2283. Fixes after rebase 04/07.
TEZ-2284. Separate TaskReporter into an interface.
TEZ-2285. Allow TaskCommunicators to indicate task/container liveness.
+ TEZ-2302. Allow TaskCommunicators to subscribe for Vertex updates.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/74c05f61/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
index 945091e..a2cd858 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.runtime.api.impl.TaskSpec;
@@ -54,4 +55,23 @@ public abstract class TaskCommunicator extends AbstractService {
public abstract InetSocketAddress getAddress();
// TODO Eventually. Add methods here to support preemption of tasks.
+
+ /**
+ * Receive notifications on vertex state changes.
+ * <p/>
+ * State changes will be received based on the registration via {@link
+ * org.apache.tez.runtime.api.InputInitializerContext#registerForVertexStateUpdates(String,
+ * java.util.Set)}. Notifications will be received for all registered state changes, and not just
+ * for the latest state update. They will be in order in which the state change occurred. </p>
+ *
+ * Extensive processing should not be performed via this method call. Instead this should just be
+ * used as a notification mechanism.
+ * <br>This method may be invoked concurrently with other invocations into the TaskCommunicator and
+ * multi-threading/concurrency implications must be considered.
+ * @param stateUpdate an event indicating the name of the vertex, and it's updated state.
+ * Additional information may be available for specific events, Look at the
+ * type hierarchy for {@link org.apache.tez.dag.api.event.VertexStateUpdate}
+ * @throws Exception
+ */
+ public abstract void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/74c05f61/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
index 0c3bac3..19caed9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
@@ -16,10 +16,12 @@ package org.apache.tez.dag.api;
import javax.annotation.Nullable;
import java.io.IOException;
+import java.util.Set;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.api.event.VertexState;
import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -48,7 +50,7 @@ public interface TaskCommunicatorContext {
void containerAlive(ContainerId containerId);
// TODO TEZ-2003 Move to vertex, taskIndex, version. Rename to taskAttempt*
- void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId);
+ void taskStartedRemotely(TezTaskAttemptID taskAttemptId, ContainerId containerId);
// TODO TEZ-2003 Move to vertex, taskIndex, version. Rename to taskAttempt*
void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason, @Nullable String diagnostics);
@@ -56,6 +58,16 @@ public interface TaskCommunicatorContext {
// TODO TEZ-2003 Move to vertex, taskIndex, version. Rename to taskAttempt*
void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason, @Nullable String diagnostics);
+ /**
+ * Register to get notifications on updates to the specified vertex. Notifications will be sent
+ * via {@link org.apache.tez.runtime.api.InputInitializer#onVertexStateUpdated(org.apache.tez.dag.api.event.VertexStateUpdate)} </p>
+ *
+ * This method can only be invoked once. Duplicate invocations will result in an error.
+ *
+ * @param vertexName the vertex name for which notifications are required.
+ * @param stateSet the set of states for which notifications are required. null implies all
+ */
+ void registerForVertexStateUpdates(String vertexName, @Nullable Set<VertexState> stateSet);
// TODO TEZ-2003 API. Should a method exist for task succeeded.
// TODO Eventually Add methods to report availability stats to the scheduler.
http://git-wip-us.apache.org/repos/asf/tez/blob/74c05f61/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 0a6710c..192e343 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -17,20 +17,22 @@
package org.apache.tez.dag.app;
+import javax.annotation.Nullable;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import com.google.common.base.Preconditions;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.common.TezUtilsInternal;
@@ -56,14 +58,12 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.runtime.api.impl.TezEvent;
import org.apache.tez.common.security.JobTokenSecretManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
@SuppressWarnings("unchecked")
@InterfaceAudience.Private
public class TaskAttemptListenerImpTezDag extends AbstractService implements
- TaskAttemptListener, TaskCommunicatorContext {
+ TaskAttemptListener {
private static final Logger LOG = LoggerFactory
.getLogger(TaskAttemptListenerImpTezDag.class);
@@ -113,7 +113,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
this.taskCommunicators = new TaskCommunicator[taskCommunicatorClassIdentifiers.length];
for (int i = 0 ; i < taskCommunicatorClassIdentifiers.length ; i++) {
- taskCommunicators[i] = createTaskCommunicator(taskCommunicatorClassIdentifiers[i]);
+ taskCommunicators[i] = createTaskCommunicator(taskCommunicatorClassIdentifiers[i], i);
}
// TODO TEZ-2118 Start using taskCommunicator indices properly
}
@@ -134,13 +134,13 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
}
- private TaskCommunicator createTaskCommunicator(String taskCommClassIdentifier) {
+ private TaskCommunicator createTaskCommunicator(String taskCommClassIdentifier, int taskCommIndex) {
if (taskCommClassIdentifier.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT)) {
LOG.info("Using Default Task Communicator");
- return new TezTaskCommunicatorImpl(this);
+ return new TezTaskCommunicatorImpl(new TaskCommunicatorContextImpl(context, this, taskCommIndex));
} else if (taskCommClassIdentifier.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT)) {
LOG.info("Using Default Local Task Communicator");
- return new TezLocalTaskCommunicatorImpl(this);
+ return new TezLocalTaskCommunicatorImpl(new TaskCommunicatorContextImpl(context, this, taskCommIndex));
} else {
LOG.info("Using TaskCommunicator: " + taskCommClassIdentifier);
Class<? extends TaskCommunicator> taskCommClazz = (Class<? extends TaskCommunicator>) ReflectionUtils
@@ -148,7 +148,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
try {
Constructor<? extends TaskCommunicator> ctor = taskCommClazz.getConstructor(TaskCommunicatorContext.class);
ctor.setAccessible(true);
- return ctor.newInstance(this);
+ return ctor.newInstance(new TaskCommunicatorContextImpl(context, this, taskCommIndex));
} catch (NoSuchMethodException e) {
throw new TezUncheckedException(e);
} catch (InvocationTargetException e) {
@@ -160,18 +160,6 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
}
}
-
- @Override
- public ApplicationAttemptId getApplicationAttemptId() {
- return context.getApplicationAttemptId();
- }
-
- @Override
- public Credentials getCredentials() {
- return context.getAppCredentials();
- }
-
- @Override
public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request)
throws IOException, TezException {
ContainerId containerId = ConverterUtils.toContainerId(request
@@ -229,30 +217,20 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
return new TaskHeartbeatResponse(false, outEvents);
}
-
- @Override
- public boolean isKnownContainer(ContainerId containerId) {
- return context.getAllContainers().get(containerId) != null;
- }
-
- @Override
public void taskAlive(TezTaskAttemptID taskAttemptId) {
taskHeartbeatHandler.pinged(taskAttemptId);
}
- @Override
public void containerAlive(ContainerId containerId) {
pingContainerHeartbeatHandler(containerId);
}
- @Override
public void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId) {
context.getEventHandler()
.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, containerId, null));
pingContainerHeartbeatHandler(containerId);
}
- @Override
public void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
String diagnostics) {
// Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
@@ -265,7 +243,6 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
taskAttemptEndReason)));
}
- @Override
public void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
String diagnostics) {
// Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
@@ -278,6 +255,11 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
taskAttemptEndReason)));
}
+ public void vertexStateUpdateNotificationReceived(VertexStateUpdate event, int taskCommIndex) throws
+ Exception {
+ taskCommunicators[taskCommIndex].onVertexStateUpdated(event);
+ }
+
/**
* Child checking whether it can commit.
@@ -287,7 +269,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
* {@link Task#canCommit(TezTaskAttemptID)} This is * a legacy from the
* centralized commit protocol handling by the JobTracker.
*/
- @Override
+// @Override
public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
LOG.info("Commit go/no-go request from " + taskAttemptId.toString());
// An attempt is asking if it can commit its output. This can be decided
http://git-wip-us.apache.org/repos/asf/tez/blob/74c05f61/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
new file mode 100644
index 0000000..3714c3c
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.Set;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.api.TaskHeartbeatRequest;
+import org.apache.tez.dag.api.TaskHeartbeatResponse;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
+import org.apache.tez.dag.app.dag.VertexStateUpdateListener;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+
+@InterfaceAudience.Private
+public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, VertexStateUpdateListener {
+
+
+ private final AppContext context;
+ private final TaskAttemptListenerImpTezDag taskAttemptListener;
+ private final int taskCommunicatorIndex;
+
+ public TaskCommunicatorContextImpl(AppContext appContext,
+ TaskAttemptListenerImpTezDag taskAttemptListener,
+ int taskCommunicatorIndex) {
+ this.context = appContext;
+ this.taskAttemptListener = taskAttemptListener;
+ this.taskCommunicatorIndex = taskCommunicatorIndex;
+ }
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return context.getApplicationAttemptId();
+ }
+
+ @Override
+ public Credentials getCredentials() {
+ return context.getAppCredentials();
+ }
+
+ @Override
+ public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
+ return taskAttemptListener.canCommit(taskAttemptId);
+ }
+
+ @Override
+ public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request) throws IOException,
+ TezException {
+ return taskAttemptListener.heartbeat(request);
+ }
+
+ @Override
+ public boolean isKnownContainer(ContainerId containerId) {
+ return context.getAllContainers().get(containerId) != null;
+ }
+
+ @Override
+ public void taskAlive(TezTaskAttemptID taskAttemptId) {
+ taskAttemptListener.taskAlive(taskAttemptId);
+ }
+
+ @Override
+ public void containerAlive(ContainerId containerId) {
+ taskAttemptListener.containerAlive(containerId);
+ }
+
+ @Override
+ public void taskStartedRemotely(TezTaskAttemptID taskAttemptId, ContainerId containerId) {
+ taskAttemptListener.taskStartedRemotely(taskAttemptId, containerId);
+ }
+
+ @Override
+ public void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
+ @Nullable String diagnostics) {
+ taskAttemptListener.taskKilled(taskAttemptId, taskAttemptEndReason, diagnostics);
+ }
+
+ @Override
+ public void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
+ @Nullable String diagnostics) {
+ taskAttemptListener.taskFailed(taskAttemptId, taskAttemptEndReason, diagnostics);
+
+ }
+
+ @Override
+ public void registerForVertexStateUpdates(String vertexName,
+ @Nullable Set<VertexState> stateSet) {
+ Preconditions.checkNotNull(vertexName, "VertexName cannot be null: " + vertexName);
+ context.getCurrentDAG().getStateChangeNotifier().registerForVertexUpdates(vertexName, stateSet, this);
+ }
+
+
+ @Override
+ public void onStateUpdated(VertexStateUpdate event) {
+ try {
+ taskAttemptListener.vertexStateUpdateNotificationReceived(event, taskCommunicatorIndex);
+ } catch (Exception e) {
+ // TODO TEZ-2003 This needs to be propagated to the DAG as a user error.
+ throw new TezUncheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/74c05f61/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
index ef4f764..1417a3b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
@@ -48,6 +48,7 @@ import org.apache.tez.dag.api.TaskHeartbeatResponse;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.runtime.api.impl.TaskSpec;
@@ -252,6 +253,11 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
return address;
}
+ @Override
+ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception {
+ // Empty. Not registering, or expecting any updates.
+ }
+
protected String getTokenIdentifier() {
return tokenIdentifier;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/74c05f61/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index 4c3426a..6d6872b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -94,4 +94,6 @@ public interface DAG {
Map<String, TezVertexID> getVertexNameIDMapping();
+ StateChangeNotifier getStateChangeNotifier();
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/74c05f61/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 9e55088..66809d4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -681,6 +681,11 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
@Override
+ public StateChangeNotifier getStateChangeNotifier() {
+ return entityUpdateTracker;
+ }
+
+ @Override
public TezCounters getAllCounters() {
readLock.lock();
[37/50] [abbrv] tez git commit: TEZ-2123. Fix component managers to
use pluggable components. Enable hybrid mode. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java
index 60782e6..12390b2 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java
@@ -59,7 +59,7 @@ public class TestLocalTaskScheduler {
TezConfiguration tezConf = new TezConfiguration();
tezConf.setInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS, MAX_TASKS);
- LocalContainerFactory containerFactory = new LocalContainerFactory(createMockAppContext());
+ LocalContainerFactory containerFactory = new LocalContainerFactory(createMockAppContext(), 1000);
HashMap<Object, Container> taskAllocations = new LinkedHashMap<Object, Container>();
PriorityBlockingQueue<TaskRequest> taskRequestQueue = new PriorityBlockingQueue<TaskRequest>();
TaskSchedulerAppCallback appClientDelegate = mock(TaskSchedulerAppCallback.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java
index 3cf4f6c..25cf4b5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java
@@ -22,6 +22,8 @@ import java.util.HashMap;
import java.util.concurrent.BlockingQueue;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -81,8 +83,12 @@ public class TestLocalTaskSchedulerService {
*/
@Test(timeout = 5000)
public void testDeallocationBeforeAllocation() {
+ AppContext appContext = mock(AppContext.class);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(ApplicationId.newInstance(10000l, 1), 1);
+ doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
MockLocalTaskSchedulerSerivce taskSchedulerService = new MockLocalTaskSchedulerSerivce
- (mock(TaskSchedulerAppCallback.class), mock(ContainerSignatureMatcher.class), "", 0, "", mock(AppContext.class));
+ (mock(TaskSchedulerAppCallback.class), mock(ContainerSignatureMatcher.class), "", 0, "", appContext);
taskSchedulerService.init(new Configuration());
taskSchedulerService.start();
@@ -105,8 +111,12 @@ public class TestLocalTaskSchedulerService {
*/
@Test(timeout = 5000)
public void testDeallocationAfterAllocation() {
+ AppContext appContext = mock(AppContext.class);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(ApplicationId.newInstance(10000l, 1), 1);
+ doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
MockLocalTaskSchedulerSerivce taskSchedulerService = new MockLocalTaskSchedulerSerivce
- (mock(TaskSchedulerAppCallback.class), mock(ContainerSignatureMatcher.class), "", 0, "", mock(AppContext.class));
+ (mock(TaskSchedulerAppCallback.class), mock(ContainerSignatureMatcher.class), "", 0, "", appContext);
taskSchedulerService.init(new Configuration());
taskSchedulerService.start();
@@ -132,13 +142,13 @@ public class TestLocalTaskSchedulerService {
String appHostName, int appHostPort, String appTrackingUrl,
AppContext appContext) {
super(appClient, containerSignatureMatcher, appHostName, appHostPort,
- appTrackingUrl, appContext);
+ appTrackingUrl, 10000l, appContext);
}
@Override
public AsyncDelegateRequestHandler createRequestHandler(Configuration conf) {
requestHandler = new MockAsyncDelegateRequestHandler(taskRequestQueue,
- new LocalContainerFactory(appContext),
+ new LocalContainerFactory(appContext, customContainerAppId),
taskAllocations,
appClientDelegate,
conf);
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
index 291e786..4ee05cc 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
@@ -89,7 +89,7 @@ public class TestTaskSchedulerEventHandler {
public MockTaskSchedulerEventHandler(AppContext appContext,
DAGClientServer clientService, EventHandler eventHandler,
ContainerSignatureMatcher containerSignatureMatcher, WebUIService webUI) {
- super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI, new String[] {});
+ super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI, new String[] {}, false);
}
@Override
@@ -162,7 +162,7 @@ public class TestTaskSchedulerEventHandler {
AMSchedulerEventTALaunchRequest lr =
new AMSchedulerEventTALaunchRequest(mockAttemptId, resource, null, mockTaskAttempt, locHint,
- priority, containerContext);
+ priority, containerContext, 0, 0, 0);
schedulerHandler.taskAllocated(mockTaskAttempt, lr, container);
assertEquals(2, mockEventHandler.events.size());
assertTrue(mockEventHandler.events.get(1) instanceof AMContainerEventAssignTA);
@@ -249,9 +249,14 @@ public class TestTaskSchedulerEventHandler {
Configuration conf = new Configuration(false);
schedulerHandler.init(conf);
schedulerHandler.start();
-
+
+ AMContainer mockAmContainer = mock(AMContainer.class);
+ when(mockAmContainer.getTaskSchedulerIdentifier()).thenReturn(0);
+ when(mockAmContainer.getContainerLauncherIdentifier()).thenReturn(0);
+ when(mockAmContainer.getTaskCommunicatorIdentifier()).thenReturn(0);
ContainerId mockCId = mock(ContainerId.class);
verify(mockTaskScheduler, times(0)).deallocateContainer((ContainerId)any());
+ when(mockAMContainerMap.get(mockCId)).thenReturn(mockAmContainer);
schedulerHandler.preemptContainer(mockCId);
verify(mockTaskScheduler, times(1)).deallocateContainer(mockCId);
assertEquals(1, mockEventHandler.events.size());
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
index d775300..ffab769 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
@@ -127,7 +127,7 @@ class TestTaskSchedulerHelpers {
EventHandler eventHandler,
TezAMRMClientAsync<CookieContainerRequest> amrmClientAsync,
ContainerSignatureMatcher containerSignatureMatcher) {
- super(appContext, null, eventHandler, containerSignatureMatcher, null, new String[]{});
+ super(appContext, null, eventHandler, containerSignatureMatcher, null, new String[]{}, false);
this.amrmClientAsync = amrmClientAsync;
this.containerSignatureMatcher = containerSignatureMatcher;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index fafbba6..bdd0f61 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.tez.common.security.JobTokenIdentifier;
import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerHeartbeatHandler;
import org.apache.tez.dag.app.ContainerContext;
@@ -104,7 +105,7 @@ public class TestAMContainer {
wc.verifyState(AMContainerState.LAUNCHING);
// 1 Launch request.
wc.verifyCountAndGetOutgoingEvents(1);
- verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
assertNull(wc.amContainer.getCurrentTaskAttempt());
// Assign task.
@@ -121,7 +122,7 @@ public class TestAMContainer {
// Once for the previous NO_TASKS, one for the actual task.
verify(wc.chh).register(wc.containerID);
ArgumentCaptor<AMContainerTask> argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
assertEquals(1, argumentCaptor.getAllValues().size());
assertEquals(wc.taskAttemptID, argumentCaptor.getAllValues().get(0).getTask().getTaskAttemptID());
assertEquals(WrappedContainer.taskPriority, argumentCaptor.getAllValues().get(0).getPriority());
@@ -131,14 +132,14 @@ public class TestAMContainer {
wc.verifyState(AMContainerState.IDLE);
wc.verifyNoOutgoingEvents();
assertNull(wc.amContainer.getCurrentTaskAttempt());
- verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID);
+ verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID, 0);
// Container completed
wc.containerCompleted();
wc.verifyHistoryStopEvent();
wc.verifyState(AMContainerState.COMPLETED);
wc.verifyNoOutgoingEvents();
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).unregister(wc.containerID);
assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
@@ -157,7 +158,7 @@ public class TestAMContainer {
wc.verifyState(AMContainerState.LAUNCHING);
// 1 Launch request.
wc.verifyCountAndGetOutgoingEvents(1);
- verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
// Container Launched
wc.containerLaunched();
@@ -172,7 +173,7 @@ public class TestAMContainer {
wc.verifyNoOutgoingEvents();
assertEquals(wc.taskAttemptID, wc.amContainer.getCurrentTaskAttempt());
ArgumentCaptor<AMContainerTask> argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
assertEquals(1, argumentCaptor.getAllValues().size());
assertEquals(wc.taskAttemptID, argumentCaptor.getAllValues().get(0).getTask().getTaskAttemptID());
@@ -180,13 +181,13 @@ public class TestAMContainer {
wc.verifyState(AMContainerState.IDLE);
wc.verifyNoOutgoingEvents();
assertNull(wc.amContainer.getCurrentTaskAttempt());
- verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID);
+ verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID, 0);
wc.containerCompleted();
wc.verifyHistoryStopEvent();
wc.verifyState(AMContainerState.COMPLETED);
wc.verifyNoOutgoingEvents();
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).unregister(wc.containerID);
assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
@@ -205,7 +206,7 @@ public class TestAMContainer {
wc.verifyState(AMContainerState.LAUNCHING);
// 1 Launch request.
wc.verifyCountAndGetOutgoingEvents(1);
- verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
assertNull(wc.amContainer.getCurrentTaskAttempt());
// Assign task.
@@ -222,7 +223,7 @@ public class TestAMContainer {
// Once for the previous NO_TASKS, one for the actual task.
verify(wc.chh).register(wc.containerID);
ArgumentCaptor<AMContainerTask> argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
assertEquals(1, argumentCaptor.getAllValues().size());
assertEquals(wc.taskAttemptID, argumentCaptor.getAllValues().get(0).getTask().getTaskAttemptID());
@@ -231,13 +232,13 @@ public class TestAMContainer {
wc.verifyState(AMContainerState.IDLE);
wc.verifyNoOutgoingEvents();
assertNull(wc.amContainer.getCurrentTaskAttempt());
- verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID);
+ verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID, 0);
TezTaskAttemptID taId2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
wc.assignTaskAttempt(taId2);
wc.verifyState(AMContainerState.RUNNING);
argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(2)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(2)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
assertEquals(2, argumentCaptor.getAllValues().size());
assertEquals(taId2, argumentCaptor.getAllValues().get(1).getTask().getTaskAttemptID());
@@ -246,14 +247,14 @@ public class TestAMContainer {
wc.verifyState(AMContainerState.IDLE);
wc.verifyNoOutgoingEvents();
assertNull(wc.amContainer.getCurrentTaskAttempt());
- verify(wc.tal).unregisterTaskAttempt(taId2);
+ verify(wc.tal).unregisterTaskAttempt(taId2, 0);
// Container completed
wc.containerCompleted();
wc.verifyHistoryStopEvent();
wc.verifyState(AMContainerState.COMPLETED);
wc.verifyNoOutgoingEvents();
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).unregister(wc.containerID);
assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
@@ -286,7 +287,7 @@ public class TestAMContainer {
wc.verifyHistoryStopEvent();
wc.verifyState(AMContainerState.COMPLETED);
wc.verifyNoOutgoingEvents();
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).unregister(wc.containerID);
assertNull(wc.amContainer.getCurrentTaskAttempt());
@@ -323,7 +324,7 @@ public class TestAMContainer {
wc.verifyHistoryStopEvent();
wc.verifyState(AMContainerState.COMPLETED);
wc.verifyNoOutgoingEvents();
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).unregister(wc.containerID);
assertNull(wc.amContainer.getCurrentTaskAttempt());
@@ -346,7 +347,7 @@ public class TestAMContainer {
wc.assignTaskAttempt(taID2);
wc.verifyState(AMContainerState.STOP_REQUESTED);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).unregister(wc.containerID);
// 1 for NM stop request. 2 TERMINATING to TaskAttempt.
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
@@ -378,13 +379,13 @@ public class TestAMContainer {
wc.launchContainer();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.verifyState(AMContainerState.LAUNCHING);
- verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
wc.assignTaskAttempt(taID2);
wc.verifyState(AMContainerState.STOP_REQUESTED);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).unregister(wc.containerID);
// 1 for NM stop request. 2 TERMINATING to TaskAttempt.
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
@@ -420,7 +421,7 @@ public class TestAMContainer {
wc.containerTimedOut();
wc.verifyState(AMContainerState.STOP_REQUESTED);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).unregister(wc.containerID);
// 1 to TA, 1 for RM de-allocate.
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
@@ -454,7 +455,7 @@ public class TestAMContainer {
wc.stopRequest();
wc.verifyState(AMContainerState.STOP_REQUESTED);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).unregister(wc.containerID);
// 1 to TA, 1 for RM de-allocate.
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
@@ -484,11 +485,11 @@ public class TestAMContainer {
wc.launchContainer();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.verifyState(AMContainerState.LAUNCHING);
- verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
wc.launchFailed();
wc.verifyState(AMContainerState.STOPPING);
- verify(wc.tal).registerRunningContainer(wc.containerID);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
@@ -537,8 +538,8 @@ public class TestAMContainer {
wc.containerCompleted();
wc.verifyState(AMContainerState.COMPLETED);
- verify(wc.tal).registerRunningContainer(wc.containerID);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
@@ -567,8 +568,8 @@ public class TestAMContainer {
wc.containerCompleted(ContainerExitStatus.DISKS_FAILED, TaskAttemptTerminationCause.NODE_DISK_ERROR);
wc.verifyState(AMContainerState.COMPLETED);
- verify(wc.tal).registerRunningContainer(wc.containerID);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
@@ -598,8 +599,8 @@ public class TestAMContainer {
wc.containerCompleted(ContainerExitStatus.ABORTED, TaskAttemptTerminationCause.NODE_FAILED);
wc.verifyState(AMContainerState.COMPLETED);
- verify(wc.tal).registerRunningContainer(wc.containerID);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
@@ -629,8 +630,8 @@ public class TestAMContainer {
wc.containerCompleted();
wc.verifyState(AMContainerState.COMPLETED);
- verify(wc.tal).registerRunningContainer(wc.containerID);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).register(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
@@ -658,8 +659,8 @@ public class TestAMContainer {
wc.containerCompleted();
wc.verifyState(AMContainerState.COMPLETED);
- verify(wc.tal).registerRunningContainer(wc.containerID);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).register(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
@@ -693,8 +694,8 @@ public class TestAMContainer {
wc.containerCompleted(ContainerExitStatus.PREEMPTED, TaskAttemptTerminationCause.EXTERNAL_PREEMPTION);
wc.verifyState(AMContainerState.COMPLETED);
- verify(wc.tal).registerRunningContainer(wc.containerID);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).register(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
@@ -730,8 +731,8 @@ public class TestAMContainer {
wc.containerCompleted(ContainerExitStatus.INVALID, TaskAttemptTerminationCause.INTERNAL_PREEMPTION);
wc.verifyState(AMContainerState.COMPLETED);
- verify(wc.tal).registerRunningContainer(wc.containerID);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).register(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
@@ -767,8 +768,8 @@ public class TestAMContainer {
wc.containerCompleted(ContainerExitStatus.DISKS_FAILED, TaskAttemptTerminationCause.NODE_DISK_ERROR);
wc.verifyState(AMContainerState.COMPLETED);
- verify(wc.tal).registerRunningContainer(wc.containerID);
- verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.tal).registerRunningContainer(wc.containerID, 0);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID, 0);
verify(wc.chh).register(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
@@ -1011,7 +1012,7 @@ public class TestAMContainer {
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
ArgumentCaptor<AMContainerTask> argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
AMContainerTask task1 = argumentCaptor.getAllValues().get(0);
assertEquals(0, task1.getAdditionalResources().size());
wc.taskAttemptSucceeded(wc.taskAttemptID);
@@ -1024,7 +1025,7 @@ public class TestAMContainer {
TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
wc.assignTaskAttempt(taID2, additionalResources, new Credentials());
argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(2)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(2)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
AMContainerTask task2 = argumentCaptor.getAllValues().get(1);
Map<String, LocalResource> pullTaskAdditionalResources = task2.getAdditionalResources();
assertEquals(2, pullTaskAdditionalResources.size());
@@ -1047,7 +1048,7 @@ public class TestAMContainer {
TezTaskAttemptID taID3 = TezTaskAttemptID.getInstance(wc.taskID, 3);
wc.assignTaskAttempt(taID3, new HashMap<String, LocalResource>(), new Credentials());
argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(3)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(3)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
AMContainerTask task3 = argumentCaptor.getAllValues().get(2);
assertEquals(0, task3.getAdditionalResources().size());
wc.taskAttemptSucceeded(taID3);
@@ -1100,7 +1101,7 @@ public class TestAMContainer {
wc.containerLaunched();
wc.assignTaskAttempt(attempt11, LRs, dag1Credentials);
argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
fetchedTask = argumentCaptor.getAllValues().get(0);
assertTrue(fetchedTask.haveCredentialsChanged());
assertNotNull(fetchedTask.getCredentials());
@@ -1109,7 +1110,7 @@ public class TestAMContainer {
wc.assignTaskAttempt(attempt12, LRs, dag1Credentials);
argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(2)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(2)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
fetchedTask = argumentCaptor.getAllValues().get(1);
assertFalse(fetchedTask.haveCredentialsChanged());
assertNull(fetchedTask.getCredentials());
@@ -1119,7 +1120,7 @@ public class TestAMContainer {
wc.setNewDAGID(dagID2);
wc.assignTaskAttempt(attempt21, LRs, null);
argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(3)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(3)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
fetchedTask = argumentCaptor.getAllValues().get(2);
assertTrue(fetchedTask.haveCredentialsChanged());
assertNull(fetchedTask.getCredentials());
@@ -1127,7 +1128,7 @@ public class TestAMContainer {
wc.assignTaskAttempt(attempt22, LRs, null);
argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(4)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(4)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
fetchedTask = argumentCaptor.getAllValues().get(3);
assertFalse(fetchedTask.haveCredentialsChanged());
assertNull(fetchedTask.getCredentials());
@@ -1137,7 +1138,7 @@ public class TestAMContainer {
wc.setNewDAGID(dagID3);
wc.assignTaskAttempt(attempt31, LRs , dag3Credentials);
argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(5)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(5)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
fetchedTask = argumentCaptor.getAllValues().get(4);
assertTrue(fetchedTask.haveCredentialsChanged());
assertNotNull(fetchedTask.getCredentials());
@@ -1147,7 +1148,7 @@ public class TestAMContainer {
wc.assignTaskAttempt(attempt32, LRs, dag1Credentials);
argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
- verify(wc.tal, times(6)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
+ verify(wc.tal, times(6)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0));
fetchedTask = argumentCaptor.getAllValues().get(5);
assertFalse(fetchedTask.haveCredentialsChanged());
assertNull(fetchedTask.getCredentials());
@@ -1200,9 +1201,10 @@ public class TestAMContainer {
chh = mock(ContainerHeartbeatHandler.class);
- InetSocketAddress addr = new InetSocketAddress("localhost", 0);
tal = mock(TaskAttemptListener.class);
- doReturn(addr).when(tal).getAddress();
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
+ doReturn(taskComm).when(tal).getTaskCommunicator(0);
dagID = TezDAGID.getInstance(applicationID, 1);
vertexID = TezVertexID.getInstance(dagID, 1);
@@ -1228,7 +1230,7 @@ public class TestAMContainer {
doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();
amContainer = new AMContainerImpl(container, chh, tal,
- new ContainerContextMatcher(), appContext);
+ new ContainerContextMatcher(), appContext, 0, 0, 0);
}
public WrappedContainer() {
@@ -1278,7 +1280,7 @@ public class TestAMContainer {
Token<JobTokenIdentifier> jobToken = mock(Token.class);
TokenCache.setSessionToken(jobToken, credentials);
amContainer.handle(new AMContainerEventLaunchRequest(containerID, vertexID,
- new ContainerContext(localResources, credentials, new HashMap<String, String>(), "")));
+ new ContainerContext(localResources, credentials, new HashMap<String, String>(), ""), 0, 0));
}
public void assignTaskAttempt(TezTaskAttemptID taID) {
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
index 61371e8..dee4541 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerHeartbeatHandler;
import org.apache.tez.dag.app.TaskAttemptListener;
@@ -43,8 +44,9 @@ public class TestAMContainerMap {
private TaskAttemptListener mockTaskAttemptListener() {
TaskAttemptListener tal = mock(TaskAttemptListener.class);
- InetSocketAddress socketAddr = new InetSocketAddress("localhost", 21000);
- doReturn(socketAddr).when(tal).getAddress();
+ TaskCommunicator taskComm = mock(TaskCommunicator.class);
+ doReturn(new InetSocketAddress("localhost", 21000)).when(taskComm).getAddress();
+ doReturn(taskComm).when(tal).getTaskCommunicator(0);
return tal;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
index d7fc5ac..52643c5 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
@@ -19,6 +19,7 @@
package org.apache.tez.examples;
import java.io.IOException;
+import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -130,7 +131,7 @@ public class JoinValidate extends TezExampleBase {
private DAG createDag(TezConfiguration tezConf, Path lhs, Path rhs, int numPartitions)
throws IOException {
- DAG dag = DAG.create("JoinValidate");
+ DAG dag = DAG.create(getDagName());
// Configuration for intermediate output - shared by Vertex1 and Vertex2
// This should only be setting selective keys from the underlying conf. Fix after there's a
@@ -147,15 +148,18 @@ public class JoinValidate extends TezExampleBase {
MRInput
.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class,
lhs.toUri().toString()).groupSplits(!isDisableSplitGrouping()).build());
+ setVertexProperties(lhsVertex, getLhsVertexProperties());
Vertex rhsVertex = Vertex.create(RHS_INPUT_NAME, ProcessorDescriptor.create(
ForwardingProcessor.class.getName())).addDataSource("rhs",
MRInput
.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class,
rhs.toUri().toString()).groupSplits(!isDisableSplitGrouping()).build());
+ setVertexProperties(rhsVertex, getRhsVertexProperties());
Vertex joinValidateVertex = Vertex.create("joinvalidate", ProcessorDescriptor.create(
JoinValidateProcessor.class.getName()), numPartitions);
+ setVertexProperties(joinValidateVertex, getValidateVertexProperties());
Edge e1 = Edge.create(lhsVertex, joinValidateVertex, edgeConf.createDefaultEdgeProperty());
Edge e2 = Edge.create(rhsVertex, joinValidateVertex, edgeConf.createDefaultEdgeProperty());
@@ -165,6 +169,30 @@ public class JoinValidate extends TezExampleBase {
return dag;
}
+ private void setVertexProperties(Vertex vertex, Map<String, String> properties) {
+ if (properties != null) {
+ for (Map.Entry<String, String> entry : properties.entrySet()) {
+ vertex.setConf(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ protected Map<String, String> getLhsVertexProperties() {
+ return null;
+ }
+
+ protected Map<String, String> getRhsVertexProperties() {
+ return null;
+ }
+
+ protected Map<String, String> getValidateVertexProperties() {
+ return null;
+ }
+
+ protected String getDagName() {
+ return "JoinValidate";
+ }
+
public static class JoinValidateProcessor extends SimpleProcessor {
private static final Logger LOG = LoggerFactory.getLogger(JoinValidateProcessor.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
index e83165b..27356bc 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
@@ -14,6 +14,8 @@
package org.apache.tez.dag.app.launcher;
+import java.net.InetSocketAddress;
+
import com.google.common.base.Preconditions;
import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
@@ -124,7 +126,8 @@ public class TezTestServiceContainerLauncher extends AbstractService implements
private RunContainerRequestProto constructRunContainerRequest(NMCommunicatorLaunchRequestEvent event) {
RunContainerRequestProto.Builder builder = RunContainerRequestProto.newBuilder();
- builder.setAmHost(tal.getAddress().getHostName()).setAmPort(tal.getAddress().getPort());
+ InetSocketAddress address = tal.getTaskCommunicator(event.getTaskCommId()).getAddress();
+ builder.setAmHost(address.getHostName()).setAmPort(address.getPort());
builder.setAppAttemptNumber(event.getContainer().getId().getApplicationAttemptId().getAttemptId());
builder.setApplicationIdString(
event.getContainer().getId().getApplicationAttemptId().getApplicationId().toString());
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
index e3c18bf..5657f86 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
@@ -14,7 +14,6 @@
package org.apache.tez.dag.app.rm;
-import java.io.IOException;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
@@ -32,25 +31,17 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.service.TezTestServiceConfConstants;
-// TODO Registration with RM - so that the AM is considered dead and restarted in the expiry interval - 10 minutes.
-
public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
private static final Log LOG = LogFactory.getLog(TezTestServiceTaskSchedulerService.class);
@@ -71,7 +62,7 @@ public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
private final ConcurrentMap<Object, ContainerId> runningTasks =
new ConcurrentHashMap<Object, ContainerId>();
- private final AMRMClientAsync<AMRMClient.ContainerRequest> amRmClient;
+ // AppIdIdentifier to avoid conflicts with other containres in the system.
// Per instance
private final int memoryPerInstance;
@@ -82,10 +73,13 @@ public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
private final Resource resourcePerContainer;
+ // Not registering with the RM. Assuming the main TezScheduler will always run (except local mode),
+ // and take care of YARN registration.
public TezTestServiceTaskSchedulerService(TaskSchedulerAppCallback appClient,
AppContext appContext,
String clientHostname, int clientPort,
String trackingUrl,
+ long customAppIdIdentifier,
Configuration conf) {
// Accepting configuration here to allow setting up fields as final
super(TezTestServiceTaskSchedulerService.class.getName());
@@ -93,7 +87,7 @@ public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
this.appClientDelegate = createAppCallbackDelegate(appClient);
this.appContext = appContext;
this.serviceHosts = new LinkedList<String>();
- this.containerFactory = new ContainerFactory(appContext);
+ this.containerFactory = new ContainerFactory(appContext, customAppIdIdentifier);
this.memoryPerInstance = conf
.getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB, -1);
@@ -123,7 +117,6 @@ public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
int memoryPerContainer = (int) (memoryPerInstance / (float) executorsPerInstance);
int coresPerContainer = (int) (coresPerInstance / (float) executorsPerInstance);
this.resourcePerContainer = Resource.newInstance(memoryPerContainer, coresPerContainer);
- this.amRmClient = TezAMRMClientAsync.createAMRMClientAsync(5000, new FakeAmRmCallbackHandler());
String[] hosts = conf.getTrimmedStrings(TezTestServiceConfConstants.TEZ_TEST_SERVICE_HOSTS);
if (hosts == null || hosts.length == 0) {
@@ -143,36 +136,8 @@ public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
}
@Override
- public void serviceInit(Configuration conf) {
- amRmClient.init(conf);
- }
-
- @Override
- public void serviceStart() {
- amRmClient.start();
- RegisterApplicationMasterResponse response;
- try {
- amRmClient.registerApplicationMaster(clientHostname, clientPort, trackingUrl);
- } catch (YarnException e) {
- throw new TezUncheckedException(e);
- } catch (IOException e) {
- throw new TezUncheckedException(e);
- }
- }
-
- @Override
public void serviceStop() {
if (!this.isStopped.getAndSet(true)) {
-
- try {
- TaskSchedulerAppCallback.AppFinalStatus status = appClientDelegate.getFinalAppStatus();
- amRmClient.unregisterApplicationMaster(status.exitStatus, status.exitMessage,
- status.postCompletionTrackingUrl);
- } catch (YarnException e) {
- throw new TezUncheckedException(e);
- } catch (IOException e) {
- throw new TezUncheckedException(e);
- }
appCallbackExecutor.shutdownNow();
}
}
@@ -264,7 +229,7 @@ public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
private ExecutorService createAppCallbackExecutorService() {
return Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
- .setNameFormat("TaskSchedulerAppCaller #%d").setDaemon(true).build());
+ .setNameFormat("TezTestTaskSchedulerAppCaller").setDaemon(true).build());
}
private TaskSchedulerAppCallback createAppCallbackDelegate(
@@ -274,7 +239,7 @@ public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
}
private String selectHost(String[] requestedHosts) {
- String host = null;
+ String host;
if (requestedHosts != null && requestedHosts.length > 0) {
Arrays.sort(requestedHosts);
host = requestedHosts[0];
@@ -287,17 +252,19 @@ public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
}
static class ContainerFactory {
- final AppContext appContext;
AtomicInteger nextId;
-
- public ContainerFactory(AppContext appContext) {
- this.appContext = appContext;
- this.nextId = new AtomicInteger(2);
+ final ApplicationAttemptId customAppAttemptId;
+
+ public ContainerFactory(AppContext appContext, long appIdLong) {
+ this.nextId = new AtomicInteger(1);
+ ApplicationId appId = ApplicationId
+ .newInstance(appIdLong, appContext.getApplicationAttemptId().getApplicationId().getId());
+ this.customAppAttemptId = ApplicationAttemptId
+ .newInstance(appId, appContext.getApplicationAttemptId().getAttemptId());
}
public Container createContainer(Resource capability, Priority priority, String hostname, int port) {
- ApplicationAttemptId appAttemptId = appContext.getApplicationAttemptId();
- ContainerId containerId = ContainerId.newInstance(appAttemptId, nextId.getAndIncrement());
+ ContainerId containerId = ContainerId.newInstance(customAppAttemptId, nextId.getAndIncrement());
NodeId nodeId = NodeId.newInstance(hostname, port);
String nodeHttpAddress = "hostname:0";
@@ -311,37 +278,4 @@ public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
return container;
}
}
-
- private static class FakeAmRmCallbackHandler implements AMRMClientAsync.CallbackHandler {
-
- @Override
- public void onContainersCompleted(List<ContainerStatus> statuses) {
-
- }
-
- @Override
- public void onContainersAllocated(List<Container> containers) {
-
- }
-
- @Override
- public void onShutdownRequest() {
-
- }
-
- @Override
- public void onNodesUpdated(List<NodeReport> updatedNodes) {
-
- }
-
- @Override
- public float getProgress() {
- return 0;
- }
-
- @Override
- public void onError(Throwable e) {
-
- }
- }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-ext-service-tests/src/test/java/org/apache/tez/examples/JoinValidateConfigured.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/examples/JoinValidateConfigured.java b/tez-ext-service-tests/src/test/java/org/apache/tez/examples/JoinValidateConfigured.java
new file mode 100644
index 0000000..e5d2e3b
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/examples/JoinValidateConfigured.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.examples;
+
+import java.util.Map;
+
+public class JoinValidateConfigured extends JoinValidate {
+
+ private final Map<String, String> lhsProps;
+ private final Map<String, String> rhsProps;
+ private final Map<String, String> validateProps;
+ private final String dagNameSuffix;
+
+ public JoinValidateConfigured(Map<String, String> lhsProps, Map<String, String> rhsProps,
+ Map<String, String> validateProps, String dagNameSuffix) {
+ this.lhsProps = lhsProps;
+ this.rhsProps = rhsProps;
+ this.validateProps = validateProps;
+ this.dagNameSuffix = dagNameSuffix;
+ }
+
+ @Override
+ protected Map<String, String> getLhsVertexProperties() {
+ return this.lhsProps;
+ }
+
+ @Override
+ protected Map<String, String> getRhsVertexProperties() {
+ return this.rhsProps;
+ }
+
+ @Override
+ protected Map<String, String> getValidateVertexProperties() {
+ return this.validateProps;
+ }
+
+ @Override
+ protected String getDagName() {
+ return "JoinValidate_" + dagNameSuffix;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/2c5faca3/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
index ae7e7f8..9c149c6 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
@@ -15,11 +15,11 @@
package org.apache.tez.tests;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.Map;
+import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -28,13 +28,14 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.tez.client.TezClient;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.app.launcher.TezTestServiceNoOpContainerLauncher;
import org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerService;
import org.apache.tez.dag.app.taskcomm.TezTestServiceTaskCommunicatorImpl;
import org.apache.tez.examples.HashJoinExample;
import org.apache.tez.examples.JoinDataGen;
-import org.apache.tez.examples.JoinValidate;
+import org.apache.tez.examples.JoinValidateConfigured;
import org.apache.tez.service.MiniTezTestServiceCluster;
import org.apache.tez.test.MiniTezCluster;
import org.junit.AfterClass;
@@ -47,23 +48,31 @@ public class TestExternalTezServices {
private static final String EXT_PUSH_ENTITY_NAME = "ExtServiceTestPush";
- private static MiniTezCluster tezCluster;
- private static MiniDFSCluster dfsCluster;
- private static MiniTezTestServiceCluster tezTestServiceCluster;
+ private static volatile MiniTezCluster tezCluster;
+ private static volatile MiniDFSCluster dfsCluster;
+ private static volatile MiniTezTestServiceCluster tezTestServiceCluster;
- private static Configuration clusterConf = new Configuration();
- private static Configuration confForJobs;
+ private static volatile Configuration clusterConf = new Configuration();
+ private static volatile Configuration confForJobs;
- private static FileSystem remoteFs;
- private static FileSystem localFs;
+ private static volatile FileSystem remoteFs;
+ private static volatile FileSystem localFs;
- private static TezClient sharedTezClient;
+ private static volatile TezClient sharedTezClient;
+
+ private static final Path SRC_DATA_DIR = new Path("/tmp/" + TestExternalTezServices.class.getSimpleName());
+ private static final Path HASH_JOIN_EXPECTED_RESULT_PATH = new Path(SRC_DATA_DIR, "expectedOutputPath");
+ private static final Path HASH_JOIN_OUTPUT_PATH = new Path(SRC_DATA_DIR, "outPath");
+
+ private static final Map<String, String> PROPS_EXT_SERVICE_PUSH = Maps.newHashMap();
+ private static final Map<String, String> PROPS_REGULAR_CONTAINERS = Maps.newHashMap();
+ private static final Map<String, String> PROPS_IN_AM = Maps.newHashMap();
private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + TestExternalTezServices.class.getName()
+ "-tmpDir";
@BeforeClass
- public static void setup() throws IOException, TezException, InterruptedException {
+ public static void setup() throws Exception {
localFs = FileSystem.getLocal(clusterConf);
@@ -108,27 +117,79 @@ public class TestExternalTezServices {
remoteFs.mkdirs(stagingDirPath);
// This is currently configured to push tasks into the Service, and then use the standard RPC
confForJobs.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
- confForJobs.set(TezConfiguration.TEZ_AM_TASK_SCHEDULERS,
+
+ confForJobs.setStrings(TezConfiguration.TEZ_AM_TASK_SCHEDULERS,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT,
+// TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT,
EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceTaskSchedulerService.class.getName());
- confForJobs.set(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHERS,
- EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceNoOpContainerLauncher.class.getName());
- confForJobs.set(TezConfiguration.TEZ_AM_TASK_COMMUNICATORS,
- EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceTaskCommunicatorImpl.class.getName());
- confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_TASK_SCHEDULER_NAME, EXT_PUSH_ENTITY_NAME);
- confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME, EXT_PUSH_ENTITY_NAME);
- confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME, EXT_PUSH_ENTITY_NAME);
+ confForJobs.setStrings(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHERS,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT,
+// TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT,
+ EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceNoOpContainerLauncher.class.getName());
+ confForJobs.setStrings(TezConfiguration.TEZ_AM_TASK_COMMUNICATORS,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT,
+// TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT,
+ EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceTaskCommunicatorImpl.class.getName());
- TezConfiguration tezConf = new TezConfiguration(confForJobs);
+ // Default all jobs to run via the service. Individual tests override this on a per vertex/dag level.
+ confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_TASK_SCHEDULER_NAME,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT);
+ confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT);
+ confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT);
+
+ // Setup various executor sets
+ PROPS_REGULAR_CONTAINERS.put(TezConfiguration.TEZ_AM_VERTEX_TASK_SCHEDULER_NAME,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT);
+ PROPS_REGULAR_CONTAINERS.put(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT);
+ PROPS_REGULAR_CONTAINERS.put(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT);
+
+ PROPS_EXT_SERVICE_PUSH.put(TezConfiguration.TEZ_AM_VERTEX_TASK_SCHEDULER_NAME, EXT_PUSH_ENTITY_NAME);
+ PROPS_EXT_SERVICE_PUSH.put(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME, EXT_PUSH_ENTITY_NAME);
+ PROPS_EXT_SERVICE_PUSH.put(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME, EXT_PUSH_ENTITY_NAME);
+
+ PROPS_IN_AM.put(TezConfiguration.TEZ_AM_VERTEX_TASK_SCHEDULER_NAME,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT);
+ PROPS_IN_AM.put(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT);
+ PROPS_IN_AM.put(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT);
+
+
+ // Create a session to use for all tests.
+ TezConfiguration tezClientConf = new TezConfiguration(confForJobs);
sharedTezClient = TezClient.create(TestExternalTezServices.class.getSimpleName() + "_session",
- tezConf, true);
+ tezClientConf, true);
sharedTezClient.start();
LOG.info("Shared TezSession started");
sharedTezClient.waitTillReady();
LOG.info("Shared TezSession ready for submission");
+ // Generate the join data set used for each run.
+ // Can a timeout be enforced here ?
+ remoteFs.mkdirs(SRC_DATA_DIR);
+ Path dataPath1 = new Path(SRC_DATA_DIR, "inPath1");
+ Path dataPath2 = new Path(SRC_DATA_DIR, "inPath2");
+ TezConfiguration tezConf = new TezConfiguration(confForJobs);
+ // Generate join data - with 2 tasks.
+ JoinDataGen dataGen = new JoinDataGen();
+ String[] dataGenArgs = new String[]{
+ dataPath1.toString(), "1048576", dataPath2.toString(), "524288",
+ HASH_JOIN_EXPECTED_RESULT_PATH.toString(), "2"};
+ assertEquals(0, dataGen.run(tezConf, dataGenArgs, sharedTezClient));
+ // Run the actual join - with 2 reducers
+ HashJoinExample joinExample = new HashJoinExample();
+ String[] args = new String[]{
+ dataPath1.toString(), dataPath2.toString(), "2", HASH_JOIN_OUTPUT_PATH.toString()};
+ assertEquals(0, joinExample.run(tezConf, args, sharedTezClient));
+
+ LOG.info("Completed generating Data - Expected Hash Result and Actual Join Result");
}
@AfterClass
@@ -156,35 +217,50 @@ public class TestExternalTezServices {
@Test(timeout = 60000)
- public void test1() throws Exception {
- Path testDir = new Path("/tmp/testHashJoinExample");
+ public void testAllInService() throws Exception {
+ int expectedExternalSubmissions = 4 + 3; //4 for 4 src files, 3 for num reducers.
+ runJoinValidate("AllInService", expectedExternalSubmissions, PROPS_EXT_SERVICE_PUSH,
+ PROPS_EXT_SERVICE_PUSH, PROPS_EXT_SERVICE_PUSH);
+ }
- remoteFs.mkdirs(testDir);
+ @Test(timeout = 60000)
+ public void testAllInContainers() throws Exception {
+ int expectedExternalSubmissions = 0; // All in containers
+ runJoinValidate("AllInContainers", expectedExternalSubmissions, PROPS_REGULAR_CONTAINERS,
+ PROPS_REGULAR_CONTAINERS, PROPS_REGULAR_CONTAINERS);
+ }
- Path dataPath1 = new Path(testDir, "inPath1");
- Path dataPath2 = new Path(testDir, "inPath2");
- Path expectedOutputPath = new Path(testDir, "expectedOutputPath");
- Path outPath = new Path(testDir, "outPath");
+ @Test(timeout = 60000)
+ public void testMixed1() throws Exception { // M-ExtService, R-containers
+ int expectedExternalSubmissions = 4 + 0; //4 for 4 src files, 3 for num reducers.
+ runJoinValidate("Mixed1", expectedExternalSubmissions, PROPS_EXT_SERVICE_PUSH,
+ PROPS_EXT_SERVICE_PUSH, PROPS_REGULAR_CONTAINERS);
+ }
- TezConfiguration tezConf = new TezConfiguration(confForJobs);
+ @Test(timeout = 60000)
+ public void testMixed2() throws Exception { // M-Containers, R-ExtService
+ int expectedExternalSubmissions = 0 + 3; //4 for 4 src files, 3 for num reducers.
+ runJoinValidate("Mixed2", expectedExternalSubmissions, PROPS_REGULAR_CONTAINERS,
+ PROPS_REGULAR_CONTAINERS, PROPS_EXT_SERVICE_PUSH);
+ }
- JoinDataGen dataGen = new JoinDataGen();
- String[] dataGenArgs = new String[]{
- dataPath1.toString(), "1048576", dataPath2.toString(), "524288",
- expectedOutputPath.toString(), "2"};
- assertEquals(0, dataGen.run(tezConf, dataGenArgs, sharedTezClient));
- HashJoinExample joinExample = new HashJoinExample();
- String[] args = new String[]{
- dataPath1.toString(), dataPath2.toString(), "2", outPath.toString()};
- assertEquals(0, joinExample.run(tezConf, args, sharedTezClient));
+ private void runJoinValidate(String name, int extExpectedCount, Map<String, String> lhsProps,
+ Map<String, String> rhsProps,
+ Map<String, String> validateProps) throws
+ Exception {
+ int externalSubmissionCount = tezTestServiceCluster.getNumSubmissions();
- JoinValidate joinValidate = new JoinValidate();
- String[] validateArgs = new String[]{
- expectedOutputPath.toString(), outPath.toString(), "3"};
+ TezConfiguration tezConf = new TezConfiguration(confForJobs);
+ JoinValidateConfigured joinValidate =
+ new JoinValidateConfigured(lhsProps, rhsProps,
+ validateProps, name);
+ String[] validateArgs = new String[]{"-disableSplitGrouping",
+ HASH_JOIN_EXPECTED_RESULT_PATH.toString(), HASH_JOIN_OUTPUT_PATH.toString(), "3"};
assertEquals(0, joinValidate.run(tezConf, validateArgs, sharedTezClient));
// Ensure this was actually submitted to the external cluster
- assertTrue(tezTestServiceCluster.getNumSubmissions() > 0);
+ assertEquals(extExpectedCount,
+ (tezTestServiceCluster.getNumSubmissions() - externalSubmissionCount));
}
}
[40/50] [abbrv] tez git commit: TEZ-2125. Create a task communicator
for local mode. Allow tasks to run in the AM. (sseth)
Posted by ss...@apache.org.
TEZ-2125. Create a task communicator for local mode. Allow tasks to run
in the AM. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/66d8892c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/66d8892c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/66d8892c
Branch: refs/heads/TEZ-2003
Commit: 66d8892c55fdc11543f6a37f178e428d5ab8a281
Parents: 2c5faca
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Feb 20 16:12:52 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:47:32 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 25 +++++---
.../dag/app/TaskAttemptListenerImpTezDag.java | 19 ++++--
.../dag/app/TezLocalTaskCommunicatorImpl.java | 46 ++++++++++++++
.../tez/dag/app/TezTaskCommunicatorImpl.java | 67 ++++++++------------
.../app/launcher/ContainerLauncherRouter.java | 17 +++--
.../app/launcher/LocalContainerLauncher.java | 31 ++++++---
.../dag/app/rm/TaskSchedulerEventHandler.java | 2 +
.../apache/tez/dag/app/MockDAGAppMaster.java | 3 +-
.../app/TestTaskAttemptListenerImplTezDag.java | 4 +-
.../tez/service/impl/ContainerRunnerImpl.java | 2 +-
.../tez/tests/TestExternalTezServices.java | 57 +++++++++++++----
.../org/apache/tez/runtime/task/TezChild.java | 34 +++++-----
13 files changed, 205 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 1a2264c..76496c9 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -5,5 +5,6 @@ ALL CHANGES:
TEZ-2117. Add a manager for ContainerLaunchers running in the AM.
TEZ-2122. Setup pluggable components at AM/Vertex level.
TEZ-2123. Fix component managers to use pluggable components. (Enable hybrid mode)
+ TEZ-2125. Create a task communicator for local mode. Allow tasks to run in the AM.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index d31cfe3..4e1657b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -463,7 +463,7 @@ public class DAGAppMaster extends AbstractService {
//service to handle requests to TaskUmbilicalProtocol
taskAttemptListener = createTaskAttemptListener(context,
- taskHeartbeatHandler, containerHeartbeatHandler, taskCommunicatorClassIdentifiers);
+ taskHeartbeatHandler, containerHeartbeatHandler, taskCommunicatorClassIdentifiers, isLocal);
addIfService(taskAttemptListener, true);
containerSignatureMatcher = createContainerSignatureMatcher();
@@ -519,7 +519,7 @@ public class DAGAppMaster extends AbstractService {
taskSchedulerEventHandler);
addIfServiceDependency(taskSchedulerEventHandler, clientRpcServer);
- this.containerLauncherRouter = createContainerLauncherRouter(conf, containerLauncherClassIdentifiers);
+ this.containerLauncherRouter = createContainerLauncherRouter(conf, containerLauncherClassIdentifiers, isLocal);
addIfService(containerLauncherRouter, true);
dispatcher.register(NMCommunicatorEventType.class, containerLauncherRouter);
@@ -1026,9 +1026,13 @@ public class DAGAppMaster extends AbstractService {
}
protected TaskAttemptListener createTaskAttemptListener(AppContext context,
- TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh, String[] taskCommunicatorClasses) {
+ TaskHeartbeatHandler thh,
+ ContainerHeartbeatHandler chh,
+ String[] taskCommunicatorClasses,
+ boolean isLocal) {
TaskAttemptListener lis =
- new TaskAttemptListenerImpTezDag(context, thh, chh,jobTokenSecretManager, taskCommunicatorClasses);
+ new TaskAttemptListenerImpTezDag(context, thh, chh, jobTokenSecretManager,
+ taskCommunicatorClasses, isLocal);
return lis;
}
@@ -1049,10 +1053,12 @@ public class DAGAppMaster extends AbstractService {
return chh;
}
- protected ContainerLauncherRouter createContainerLauncherRouter(Configuration conf, String []containerLauncherClasses) throws
+ protected ContainerLauncherRouter createContainerLauncherRouter(Configuration conf,
+ String[] containerLauncherClasses,
+ boolean isLocal) throws
UnknownHostException {
- return new ContainerLauncherRouter(conf, context, taskAttemptListener, workingDirectory, containerLauncherClasses);
-
+ return new ContainerLauncherRouter(conf, context, taskAttemptListener, workingDirectory,
+ containerLauncherClasses, isLocal);
}
public ApplicationId getAppID() {
@@ -2294,9 +2300,8 @@ public class DAGAppMaster extends AbstractService {
StringBuilder sb = new StringBuilder();
sb.append("AM Level configured ").append(component).append(": ");
for (int i = 0; i < classIdentifiers.length; i++) {
- sb.append("[").append(i).append(":").append(map.inverse().get(i)).append(":")
- .append(taskSchedulers.inverse().get(i)).append(
- "]");
+ sb.append("[").append(i).append(":").append(map.inverse().get(i))
+ .append(":").append(classIdentifiers[i]).append("]");
if (i != classIdentifiers.length - 1) {
sb.append(",");
}
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index adb21e7..5435633 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -20,7 +20,6 @@ package org.apache.tez.dag.app;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
-import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -89,13 +88,20 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
// TODO TEZ-2003 pre-merge. Remove reference to JobTokenSecretManager.
JobTokenSecretManager jobTokenSecretManager,
- String [] taskCommunicatorClassIdentifiers) {
+ String [] taskCommunicatorClassIdentifiers,
+ boolean isPureLocalMode) {
super(TaskAttemptListenerImpTezDag.class.getName());
this.context = context;
this.taskHeartbeatHandler = thh;
this.containerHeartbeatHandler = chh;
if (taskCommunicatorClassIdentifiers == null || taskCommunicatorClassIdentifiers.length == 0) {
- taskCommunicatorClassIdentifiers = new String[] {TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT};
+ if (isPureLocalMode) {
+ taskCommunicatorClassIdentifiers =
+ new String[]{TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT};
+ } else {
+ taskCommunicatorClassIdentifiers =
+ new String[]{TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT};
+ }
}
this.taskCommunicators = new TaskCommunicator[taskCommunicatorClassIdentifiers.length];
for (int i = 0 ; i < taskCommunicatorClassIdentifiers.length ; i++) {
@@ -121,11 +127,12 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
private TaskCommunicator createTaskCommunicator(String taskCommClassIdentifier) {
- if (taskCommClassIdentifier.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT) ||
- taskCommClassIdentifier
- .equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT)) {
+ if (taskCommClassIdentifier.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT)) {
LOG.info("Using Default Task Communicator");
return new TezTaskCommunicatorImpl(this);
+ } else if (taskCommClassIdentifier.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT)) {
+ LOG.info("Using Default Local Task Communicator");
+ return new TezLocalTaskCommunicatorImpl(this);
} else {
LOG.info("Using TaskCommunicator: " + taskCommClassIdentifier);
Class<? extends TaskCommunicator> taskCommClazz = (Class<? extends TaskCommunicator>) ReflectionUtils
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-dag/src/main/java/org/apache/tez/dag/app/TezLocalTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezLocalTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezLocalTaskCommunicatorImpl.java
new file mode 100644
index 0000000..3704cc4
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezLocalTaskCommunicatorImpl.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.api.TezUncheckedException;
+
+public class TezLocalTaskCommunicatorImpl extends TezTaskCommunicatorImpl {
+
+ private static final Log LOG = LogFactory.getLog(TezLocalTaskCommunicatorImpl.class);
+
+ public TezLocalTaskCommunicatorImpl(
+ TaskCommunicatorContext taskCommunicatorContext) {
+ super(taskCommunicatorContext);
+ }
+
+ @Override
+ protected void startRpcServer() {
+ try {
+ this.address = new InetSocketAddress(InetAddress.getLocalHost(), 0);
+ } catch (UnknownHostException e) {
+ throw new TezUncheckedException(e);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not starting TaskAttemptListener RPC in LocalMode");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
index 258c927..0bf1b5d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
@@ -15,10 +15,8 @@
package org.apache.tez.dag.app;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
-import java.net.UnknownHostException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -76,7 +74,7 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
private final TezTaskUmbilicalProtocol taskUmbilical;
private final String tokenIdentifier;
private final Token<JobTokenIdentifier> sessionToken;
- private InetSocketAddress address;
+ protected InetSocketAddress address;
private Server server;
public static final class ContainerInfo {
@@ -120,10 +118,8 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
this.sessionToken = TokenCache.getSessionToken(taskCommunicatorContext.getCredentials());
}
-
@Override
public void serviceStart() {
-
startRpcServer();
}
@@ -134,43 +130,32 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
protected void startRpcServer() {
Configuration conf = getConfig();
- if (!conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT)) {
- try {
- JobTokenSecretManager jobTokenSecretManager =
- new JobTokenSecretManager();
- jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken);
-
- server = new RPC.Builder(conf)
- .setProtocol(TezTaskUmbilicalProtocol.class)
- .setBindAddress("0.0.0.0")
- .setPort(0)
- .setInstance(taskUmbilical)
- .setNumHandlers(
- conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
- TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT))
- .setSecretManager(jobTokenSecretManager).build();
-
- // Enable service authorization?
- if (conf.getBoolean(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
- false)) {
- refreshServiceAcls(conf, new TezAMPolicyProvider());
- }
-
- server.start();
- this.address = NetUtils.getConnectAddress(server);
- } catch (IOException e) {
- throw new TezUncheckedException(e);
- }
- } else {
- try {
- this.address = new InetSocketAddress(InetAddress.getLocalHost(), 0);
- } catch (UnknownHostException e) {
- throw new TezUncheckedException(e);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Not starting TaskAttemptListener RPC in LocalMode");
+ try {
+ JobTokenSecretManager jobTokenSecretManager =
+ new JobTokenSecretManager();
+ jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken);
+
+ server = new RPC.Builder(conf)
+ .setProtocol(TezTaskUmbilicalProtocol.class)
+ .setBindAddress("0.0.0.0")
+ .setPort(0)
+ .setInstance(taskUmbilical)
+ .setNumHandlers(
+ conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
+ TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT))
+ .setSecretManager(jobTokenSecretManager).build();
+
+ // Enable service authorization?
+ if (conf.getBoolean(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
+ false)) {
+ refreshServiceAcls(conf, new TezAMPolicyProvider());
}
+
+ server.start();
+ this.address = NetUtils.getConnectAddress(server);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
index 4f9b5bf..70b0cbc 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.common.ReflectionUtils;
-import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
@@ -49,17 +48,24 @@ public class ContainerLauncherRouter extends AbstractService
public ContainerLauncherRouter(Configuration conf, AppContext context,
TaskAttemptListener taskAttemptListener,
String workingDirectory,
- String[] containerLauncherClassIdentifiers) throws UnknownHostException {
+ String[] containerLauncherClassIdentifiers,
+ boolean isPureLocalMode) throws UnknownHostException {
super(ContainerLauncherRouter.class.getName());
if (containerLauncherClassIdentifiers == null || containerLauncherClassIdentifiers.length == 0) {
- containerLauncherClassIdentifiers = new String[] {TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT};
+ if (isPureLocalMode) {
+ containerLauncherClassIdentifiers =
+ new String[]{TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT};
+ } else {
+ containerLauncherClassIdentifiers =
+ new String[]{TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT};
+ }
}
containerLaunchers = new ContainerLauncher[containerLauncherClassIdentifiers.length];
for (int i = 0; i < containerLauncherClassIdentifiers.length; i++) {
containerLaunchers[i] = createContainerLauncher(containerLauncherClassIdentifiers[i], context,
- taskAttemptListener, workingDirectory, conf);
+ taskAttemptListener, workingDirectory, isPureLocalMode, conf);
}
}
@@ -67,6 +73,7 @@ public class ContainerLauncherRouter extends AbstractService
AppContext context,
TaskAttemptListener taskAttemptListener,
String workingDirectory,
+ boolean isPureLocalMode,
Configuration conf) throws
UnknownHostException {
if (containerLauncherClassIdentifier.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT)) {
@@ -76,7 +83,7 @@ public class ContainerLauncherRouter extends AbstractService
.equals(TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT)) {
LOG.info("Creating LocalContainerLauncher");
return
- new LocalContainerLauncher(context, taskAttemptListener, workingDirectory);
+ new LocalContainerLauncher(context, taskAttemptListener, workingDirectory, isPureLocalMode);
} else {
LOG.info("Creating container launcher : " + containerLauncherClassIdentifier);
Class<? extends ContainerLauncher> containerLauncherClazz =
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 9a38732..18b2e35 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -36,6 +36,7 @@ import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
@@ -90,9 +91,10 @@ public class LocalContainerLauncher extends AbstractService implements
private final AtomicBoolean serviceStopped = new AtomicBoolean(false);
private final String workingDirectory;
private final TaskAttemptListener tal;
- private final Map<String, String> localEnv = new HashMap<String, String>();
+ private final Map<String, String> localEnv;
private final ExecutionContext executionContext;
private int numExecutors;
+ private final boolean isPureLocalMode;
private final ConcurrentHashMap<ContainerId, ListenableFuture<TezChild.ContainerExecutionResult>>
runningContainers =
@@ -112,16 +114,26 @@ public class LocalContainerLauncher extends AbstractService implements
public LocalContainerLauncher(AppContext context,
TaskAttemptListener taskAttemptListener,
- String workingDirectory) throws UnknownHostException {
+ String workingDirectory,
+ boolean isPureLocalMode) throws UnknownHostException {
super(LocalContainerLauncher.class.getName());
this.context = context;
this.tal = taskAttemptListener;
this.workingDirectory = workingDirectory;
- AuxiliaryServiceHelper.setServiceDataIntoEnv(
- ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, ByteBuffer.allocate(4).putInt(0), localEnv);
- executionContext = new ExecutionContextImpl(InetAddress.getLocalHost().getHostName());
- // User cannot be set here since it isn't available till a DAG is running.
+ this.isPureLocalMode = isPureLocalMode;
+ if (isPureLocalMode) {
+ localEnv = Maps.newHashMap();
+ AuxiliaryServiceHelper.setServiceDataIntoEnv(
+ ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, ByteBuffer.allocate(4).putInt(0), localEnv);
+ } else {
+ localEnv = System.getenv();
+ }
+
+ // Check if the hostname is set in the environment before overriding it.
+ String host = isPureLocalMode ? InetAddress.getLocalHost().getHostName() :
+ System.getenv(Environment.NM_HOST.name());
+ executionContext = new ExecutionContextImpl(host);
}
@Override
@@ -338,7 +350,9 @@ public class LocalContainerLauncher extends AbstractService implements
InterruptedException, TezException, IOException {
Map<String, String> containerEnv = new HashMap<String, String>();
containerEnv.putAll(localEnv);
- containerEnv.put(Environment.USER.name(), context.getUser());
+ // Use the user from env if it's available.
+ String user = isPureLocalMode ? System.getenv(Environment.USER.name()) : context.getUser();
+ containerEnv.put(Environment.USER.name(), user);
long memAvailable;
synchronized (this) { // needed to fix findbugs Inconsistent synchronization warning
@@ -347,8 +361,7 @@ public class LocalContainerLauncher extends AbstractService implements
TezChild tezChild =
TezChild.newTezChild(defaultConf, null, 0, containerId.toString(), tokenIdentifier,
attemptNumber, localDirs, workingDirectory, containerEnv, "", executionContext, credentials,
- memAvailable, context.getUser());
- tezChild.setUmbilical(tezTaskUmbilicalProtocol);
+ memAvailable, context.getUser(), tezTaskUmbilicalProtocol);
return tezChild;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 5a0ace8..5a8e9fe 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -434,6 +434,8 @@ public class TaskSchedulerEventHandler extends AbstractService
} else {
customAppIdIdentifier = SCHEDULER_APP_ID_BASE + (j++ * SCHEDULER_APP_ID_INCREMENT);
}
+ LOG.info("ClusterIdentifier for TaskScheduler [" + i + ":" + taskSchedulerClasses[i] + "]=" +
+ customAppIdIdentifier);
taskSchedulers[i] = createTaskScheduler(host, port,
trackingUrl, appContext, taskSchedulerClasses[i], customAppIdIdentifier);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index a466bc6..73e93c5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -516,7 +516,8 @@ public class MockDAGAppMaster extends DAGAppMaster {
// use mock container launcher for tests
@Override
protected ContainerLauncherRouter createContainerLauncherRouter(final Configuration conf,
- String[] containerLaunchers)
+ String[] containerLaunchers,
+ boolean isLocal)
throws UnknownHostException {
return new ContainerLauncherRouter(containerLauncher);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
index 318ee4e..361565a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
@@ -73,7 +73,7 @@ public class TestTaskAttemptListenerImplTezDag {
TaskAttemptListenerImpTezDag taskAttemptListener =
new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
- mock(ContainerHeartbeatHandler.class), null, null);
+ mock(ContainerHeartbeatHandler.class), null, null, false);
TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
@@ -153,7 +153,7 @@ public class TestTaskAttemptListenerImplTezDag {
TaskAttemptListenerImpTezDag taskAttemptListener =
new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
- mock(ContainerHeartbeatHandler.class), null, null);
+ mock(ContainerHeartbeatHandler.class), null, null, false);
TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
index 4a6ce33..25d6030 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
@@ -282,7 +282,7 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
request.getContainerIdString(),
request.getTokenIdentifier(), request.getAppAttemptNumber(), workingDir, localDirs,
envMap, objectRegistry, pid,
- executionContext, credentials, memoryAvailable, request.getUser());
+ executionContext, credentials, memoryAvailable, request.getUser(), null);
ContainerExecutionResult result = tezChild.run();
LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" +
sw.stop().elapsedMillis());
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
index 9c149c6..01c2080 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
@@ -40,6 +40,7 @@ import org.apache.tez.service.MiniTezTestServiceCluster;
import org.apache.tez.test.MiniTezCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
public class TestExternalTezServices {
@@ -120,26 +121,23 @@ public class TestExternalTezServices {
confForJobs.setStrings(TezConfiguration.TEZ_AM_TASK_SCHEDULERS,
TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT,
-// TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT,
EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceTaskSchedulerService.class.getName());
confForJobs.setStrings(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHERS,
TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT,
-// TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT,
EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceNoOpContainerLauncher.class.getName());
confForJobs.setStrings(TezConfiguration.TEZ_AM_TASK_COMMUNICATORS,
TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT,
-// TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT,
+ TezConstants.TEZ_AM_SERVICE_PLUGINS_LOCAL_MODE_NAME_DEFAULT,
EXT_PUSH_ENTITY_NAME + ":" + TezTestServiceTaskCommunicatorImpl.class.getName());
// Default all jobs to run via the service. Individual tests override this on a per vertex/dag level.
- confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_TASK_SCHEDULER_NAME,
- TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT);
- confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME,
- TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT);
- confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME,
- TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT);
+ confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_TASK_SCHEDULER_NAME, EXT_PUSH_ENTITY_NAME);
+ confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME, EXT_PUSH_ENTITY_NAME);
+ confForJobs.set(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME, EXT_PUSH_ENTITY_NAME);
// Setup various executor sets
PROPS_REGULAR_CONTAINERS.put(TezConfiguration.TEZ_AM_VERTEX_TASK_SCHEDULER_NAME,
@@ -232,18 +230,55 @@ public class TestExternalTezServices {
@Test(timeout = 60000)
public void testMixed1() throws Exception { // M-ExtService, R-containers
- int expectedExternalSubmissions = 4 + 0; //4 for 4 src files, 3 for num reducers.
+ int expectedExternalSubmissions = 4 + 0; //4 for 4 src files, 0 for num reducers.
runJoinValidate("Mixed1", expectedExternalSubmissions, PROPS_EXT_SERVICE_PUSH,
PROPS_EXT_SERVICE_PUSH, PROPS_REGULAR_CONTAINERS);
}
@Test(timeout = 60000)
public void testMixed2() throws Exception { // M-Containers, R-ExtService
- int expectedExternalSubmissions = 0 + 3; //4 for 4 src files, 3 for num reducers.
+ int expectedExternalSubmissions = 0 + 3; // 3 for num reducers.
runJoinValidate("Mixed2", expectedExternalSubmissions, PROPS_REGULAR_CONTAINERS,
PROPS_REGULAR_CONTAINERS, PROPS_EXT_SERVICE_PUSH);
}
+ @Test(timeout = 60000)
+ public void testMixed3() throws Exception { // M - service, R-AM
+ int expectedExternalSubmissions = 4 + 0; //4 for 4 src files, 0 for num reducers (in-AM).
+ runJoinValidate("Mixed3", expectedExternalSubmissions, PROPS_EXT_SERVICE_PUSH,
+ PROPS_EXT_SERVICE_PUSH, PROPS_IN_AM);
+ }
+
+ @Test(timeout = 60000)
+ public void testMixed4() throws Exception { // M - containers, R-AM
+ int expectedExternalSubmissions = 0 + 0; // Nothing in external service.
+ runJoinValidate("Mixed4", expectedExternalSubmissions, PROPS_REGULAR_CONTAINERS,
+ PROPS_REGULAR_CONTAINERS, PROPS_IN_AM);
+ }
+
+ @Test(timeout = 60000)
+ public void testMixed5() throws Exception { // M1 - containers, M2-extservice, R-AM
+ int expectedExternalSubmissions = 2 + 0; // 2 for M2
+ runJoinValidate("Mixed5", expectedExternalSubmissions, PROPS_REGULAR_CONTAINERS,
+ PROPS_EXT_SERVICE_PUSH, PROPS_IN_AM);
+ }
+
+
+ @Ignore // Re-activate this after the AM registers the shuffle token with the launcher.
+ @Test(timeout = 60000)
+ public void testMixed6() throws Exception { // M - AM, R - Service
+ int expectedExternalSubmissions = 0 + 3; // 3 for R in service
+ runJoinValidate("Mixed6", expectedExternalSubmissions, PROPS_IN_AM,
+ PROPS_IN_AM, PROPS_EXT_SERVICE_PUSH);
+ }
+
+ @Test(timeout = 60000)
+ public void testMixed7() throws Exception { // M - AM, R - Containers
+ int expectedExternalSubmissions = 0; // Nothing in ext service
+ runJoinValidate("Mixed7", expectedExternalSubmissions, PROPS_IN_AM,
+ PROPS_IN_AM, PROPS_REGULAR_CONTAINERS);
+ }
+
private void runJoinValidate(String name, int extExpectedCount, Map<String, String> lhsProps,
Map<String, String> rhsProps,
http://git-wip-us.apache.org/repos/asf/tez/blob/66d8892c/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 3cba3ce..7615f08 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -64,6 +64,7 @@ import org.apache.tez.dag.utils.RelocalizationUtils;
import org.apache.tez.runtime.api.ExecutionContext;
import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.runtime.api.impl.TezUmbilical;
import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,7 +94,6 @@ public class TezChild {
private final int amHeartbeatInterval;
private final long sendCounterInterval;
private final int maxEventsToGet;
- private final boolean isLocal;
private final String workingDir;
private final ListeningExecutorService executor;
@@ -108,9 +108,10 @@ public class TezChild {
private final String user;
private Multimap<String, String> startedInputsMap = HashMultimap.create();
+ private final boolean ownUmbilical;
+ private final TezTaskUmbilicalProtocol umbilical;
private TaskReporter taskReporter;
- private TezTaskUmbilicalProtocol umbilical;
private int taskCount = 0;
private TezVertexID lastVertexID;
@@ -119,7 +120,7 @@ public class TezChild {
Map<String, String> serviceProviderEnvMap,
ObjectRegistryImpl objectRegistry, String pid,
ExecutionContext executionContext,
- Credentials credentials, long memAvailable, String user)
+ Credentials credentials, long memAvailable, String user, TezTaskUmbilicalProtocol umbilical)
throws IOException, InterruptedException {
this.defaultConf = conf;
this.containerIdString = containerIdentifier;
@@ -133,6 +134,8 @@ public class TezChild {
this.memAvailable = memAvailable;
this.user = user;
+ LOG.info("TezChild created with umbilical: " + umbilical);
+
getTaskMaxSleepTime = defaultConf.getInt(
TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX,
TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX_DEFAULT);
@@ -161,25 +164,27 @@ public class TezChild {
}
}
- this.isLocal = defaultConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
- TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
UserGroupInformation taskOwner = UserGroupInformation.createRemoteUser(tokenIdentifier);
Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
serviceConsumerMetadata.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
TezCommonUtils.convertJobTokenToBytes(jobToken));
- if (!isLocal) {
+ if (umbilical == null) {
final InetSocketAddress address = NetUtils.createSocketAddrForHost(host, port);
SecurityUtil.setTokenService(jobToken, address);
taskOwner.addToken(jobToken);
- umbilical = taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() {
+ this.umbilical = taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() {
@Override
public TezTaskUmbilicalProtocol run() throws Exception {
return RPC.getProxy(TezTaskUmbilicalProtocol.class,
TezTaskUmbilicalProtocol.versionID, address, defaultConf);
}
});
+ ownUmbilical = true;
+ } else {
+ this.umbilical = umbilical;
+ ownUmbilical = false;
}
}
@@ -353,7 +358,7 @@ public class TezChild {
if (taskReporter != null) {
taskReporter.shutdown();
}
- if (!isLocal) {
+ if (ownUmbilical) {
RPC.stopProxy(umbilical);
// TODO Temporary change. Revert. Ideally, move this over to the main method in TezChild if possible.
// LogManager.shutdown();
@@ -361,12 +366,6 @@ public class TezChild {
}
}
- public void setUmbilical(TezTaskUmbilicalProtocol tezTaskUmbilicalProtocol){
- if(tezTaskUmbilicalProtocol != null){
- this.umbilical = tezTaskUmbilicalProtocol;
- }
- }
-
public static class ContainerExecutionResult {
public static enum ExitStatus {
SUCCESS(0),
@@ -412,7 +411,8 @@ public class TezChild {
public static TezChild newTezChild(Configuration conf, String host, int port, String containerIdentifier,
String tokenIdentifier, int attemptNumber, String[] localDirs, String workingDirectory,
Map<String, String> serviceProviderEnvMap, @Nullable String pid,
- ExecutionContext executionContext, Credentials credentials, long memAvailable, String user)
+ ExecutionContext executionContext, Credentials credentials, long memAvailable, String user,
+ TezTaskUmbilicalProtocol tezUmbilical)
throws IOException, InterruptedException, TezException {
// Pull in configuration specified for the session.
@@ -425,7 +425,7 @@ public class TezChild {
return new TezChild(conf, host, port, containerIdentifier, tokenIdentifier,
attemptNumber, workingDirectory, localDirs, serviceProviderEnvMap, objectRegistry, pid,
- executionContext, credentials, memAvailable, user);
+ executionContext, credentials, memAvailable, user, tezUmbilical);
}
public static void main(String[] args) throws IOException, InterruptedException, TezException {
@@ -459,7 +459,7 @@ public class TezChild {
tokenIdentifier, attemptNumber, localDirs, System.getenv(Environment.PWD.name()),
System.getenv(), pid, new ExecutionContextImpl(System.getenv(Environment.NM_HOST.name())),
credentials, Runtime.getRuntime().maxMemory(), System
- .getenv(ApplicationConstants.Environment.USER.toString()));
+ .getenv(ApplicationConstants.Environment.USER.toString()), null);
tezChild.run();
}
[39/50] [abbrv] tez git commit: TEZ-2138. Fix minor bugs in adding
default scheduler, getting launchers. (sseth)
Posted by ss...@apache.org.
TEZ-2138. Fix minor bugs in adding default scheduler, getting launchers. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/d78ffbf8
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/d78ffbf8
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/d78ffbf8
Branch: refs/heads/TEZ-2003
Commit: d78ffbf8916a12ecc771d89c0093769e2c8158e0
Parents: 86ff66a
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Feb 23 20:53:24 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:47:32 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../src/main/java/org/apache/tez/dag/app/DAGAppMaster.java | 2 +-
.../java/org/apache/tez/dag/app/dag/impl/VertexImpl.java | 4 ++++
.../org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java | 1 +
.../dag/app/launcher/TezTestServiceContainerLauncher.java | 8 +++++---
5 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/d78ffbf8/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 4b0a12b..4377f57 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -7,5 +7,6 @@ ALL CHANGES:
TEZ-2123. Fix component managers to use pluggable components. (Enable hybrid mode)
TEZ-2125. Create a task communicator for local mode. Allow tasks to run in the AM.
TEZ-2131. Add additional tests for tasks running in the AM.
+ TEZ-2138. Fix minor bugs in adding default scheduler, getting launchers.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/d78ffbf8/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 4e1657b..abc2fa4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -1495,7 +1495,7 @@ public class DAGAppMaster extends AbstractService {
@Override
public Integer getContainerLauncherIdentifier(String name) {
- return taskCommunicators.get(name);
+ return containerLaunchers.get(name);
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/d78ffbf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 58607e6..0705d37 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -953,6 +953,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
.get(TezConfiguration.TEZ_AM_VERTEX_TASK_COMMUNICATOR_NAME, tezDefaultComponentName);
String containerLauncherName = vertexConf
.get(TezConfiguration.TEZ_AM_VERTEX_CONTAINER_LAUNCHER_NAME, tezDefaultComponentName);
+
+ LOG.info("Vertex: " + logIdentifier + " configured with TaskScheduler=" + taskSchedulerName +
+ ", ContainerLauncher=" + containerLauncherName + ", TaskComm=" + taskCommName);
+
taskSchedulerIdentifier = appContext.getTaskScheduerIdentifier(taskSchedulerName);
taskCommunicatorIdentifier = appContext.getTaskCommunicatorIdentifier(taskCommName);
containerLauncherIdentifier = appContext.getContainerLauncherIdentifier(containerLauncherName);
http://git-wip-us.apache.org/repos/asf/tez/blob/d78ffbf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 5a8e9fe..8e5fc71 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -174,6 +174,7 @@ public class TaskSchedulerEventHandler extends AbstractService
for (int i = 0 ; i < schedulerClasses.length ; i++) { // Copy over the rest.
this.taskSchedulerClasses[i] = schedulerClasses[i];
}
+ this.taskSchedulerClasses[foundYarnTaskSchedulerIndex] = TezConstants.TEZ_AM_SERVICE_PLUGINS_NAME_DEFAULT;
} else {
this.taskSchedulerClasses = schedulerClasses;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/d78ffbf8/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
index 27356bc..cb6308c 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.tez.dag.app.AppContext;
@@ -54,6 +55,7 @@ public class TezTestServiceContainerLauncher extends AbstractService implements
private final int servicePort;
private final TezTestServiceCommunicator communicator;
private final Clock clock;
+ private final ApplicationAttemptId appAttemptId;
// Configuration passed in here to set up final parameters
@@ -70,6 +72,7 @@ public class TezTestServiceContainerLauncher extends AbstractService implements
this.communicator = new TezTestServiceCommunicator(numThreads);
this.context = appContext;
this.tokenIdentifier = context.getApplicationID().toString();
+ this.appAttemptId = appContext.getApplicationAttemptId();
this.tal = tal;
}
@@ -128,9 +131,8 @@ public class TezTestServiceContainerLauncher extends AbstractService implements
RunContainerRequestProto.Builder builder = RunContainerRequestProto.newBuilder();
InetSocketAddress address = tal.getTaskCommunicator(event.getTaskCommId()).getAddress();
builder.setAmHost(address.getHostName()).setAmPort(address.getPort());
- builder.setAppAttemptNumber(event.getContainer().getId().getApplicationAttemptId().getAttemptId());
- builder.setApplicationIdString(
- event.getContainer().getId().getApplicationAttemptId().getApplicationId().toString());
+ builder.setAppAttemptNumber(appAttemptId.getAttemptId());
+ builder.setApplicationIdString(appAttemptId.getApplicationId().toString());
builder.setTokenIdentifier(tokenIdentifier);
builder.setContainerIdString(event.getContainer().getId().toString());
builder.setCredentialsBinary(
[10/50] [abbrv] tez git commit: TEZ-714. OutputCommitters should not
run in the main AM dispatcher thread (zjffdu)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 82f7a33..b4b8062 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -24,6 +24,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
@@ -31,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
@@ -94,6 +96,7 @@ import org.apache.tez.dag.app.ContainerContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.DAGState;
import org.apache.tez.dag.app.dag.RootInputInitializerManager;
import org.apache.tez.dag.app.dag.StateChangeNotifier;
import org.apache.tez.dag.app.dag.Task;
@@ -102,6 +105,7 @@ import org.apache.tez.dag.app.dag.TaskTerminationCause;
import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.VertexState;
import org.apache.tez.dag.app.dag.VertexTerminationCause;
+import org.apache.tez.dag.app.dag.event.CallableEvent;
import org.apache.tez.dag.app.dag.event.DAGEvent;
import org.apache.tez.dag.app.dag.event.DAGEventDiagnosticsUpdate;
import org.apache.tez.dag.app.dag.event.DAGEventType;
@@ -117,6 +121,7 @@ import org.apache.tez.dag.app.dag.event.TaskEventRecoverTask;
import org.apache.tez.dag.app.dag.event.TaskEventTermination;
import org.apache.tez.dag.app.dag.event.TaskEventType;
import org.apache.tez.dag.app.dag.event.VertexEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventCommitCompleted;
import org.apache.tez.dag.app.dag.event.VertexEventInputDataInformation;
import org.apache.tez.dag.app.dag.event.VertexEventManagerUserCodeError;
import org.apache.tez.dag.app.dag.event.VertexEventNullEdgeInitialized;
@@ -179,6 +184,9 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multiset;
import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
import org.apache.tez.state.OnStateChangedCallback;
import org.apache.tez.state.StateMachineTez;
@@ -255,6 +263,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private static final SourceTaskAttemptCompletedEventTransition
SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION =
new SourceTaskAttemptCompletedEventTransition();
+ private static final CommitCompletedTransition
+ COMMIT_COMPLETED_TRANSITION =
+ new CommitCompletedTransition();
private static final VertexStateChangedCallback STATE_CHANGED_CALLBACK =
new VertexStateChangedCallback();
@@ -268,6 +279,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
LegacySpeculator speculator;
+ @VisibleForTesting
+ Map<String, ListenableFuture<Void>> commitFutures = new ConcurrentHashMap<String, ListenableFuture<Void>>();
+
protected static final
StateMachineFactory<VertexImpl, VertexState, VertexEventType, VertexEvent>
stateMachineFactory
@@ -445,6 +459,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
.addTransition
(VertexState.RUNNING,
EnumSet.of(VertexState.RUNNING,
+ VertexState.COMMITTING,
VertexState.SUCCEEDED, VertexState.TERMINATING, VertexState.FAILED,
VertexState.ERROR),
VertexEventType.V_TASK_COMPLETED,
@@ -473,6 +488,38 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
VertexEventType.V_ROUTE_EVENT,
ROUTE_EVENT_TRANSITION)
+ // Transitions from COMMITTING state.
+ .addTransition(
+ VertexState.COMMITTING,
+ EnumSet.of(VertexState.COMMITTING, VertexState.TERMINATING,
+ VertexState.SUCCEEDED, VertexState.FAILED),
+ VertexEventType.V_COMMIT_COMPLETED,
+ COMMIT_COMPLETED_TRANSITION)
+ .addTransition(
+ VertexState.COMMITTING,
+ VertexState.TERMINATING,
+ VertexEventType.V_TERMINATE,
+ new VertexKilledWhileCommittingTransition())
+ .addTransition(
+ VertexState.COMMITTING,
+ VertexState.ERROR,
+ VertexEventType.V_INTERNAL_ERROR,
+ INTERNAL_ERROR_TRANSITION)
+ .addTransition(
+ VertexState.COMMITTING,
+ EnumSet.of(VertexState.COMMITTING, VertexState.TERMINATING),
+ VertexEventType.V_ROUTE_EVENT,
+ ROUTE_EVENT_TRANSITION)
+ .addTransition(
+ VertexState.COMMITTING,
+ VertexState.TERMINATING,
+ VertexEventType.V_TASK_RESCHEDULED,
+ new TaskRescheduledWhileCommittingTransition())
+ .addTransition(VertexState.COMMITTING,
+ EnumSet.of(VertexState.TERMINATING),
+ VertexEventType.V_MANAGER_USER_CODE_ERROR,
+ new VertexManagerUserCodeErrorTransition())
+
// Transitions from TERMINATING state.
.addTransition
(VertexState.TERMINATING,
@@ -483,6 +530,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
VertexState.TERMINATING,
VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
+ .addTransition(
+ VertexState.TERMINATING,
+ EnumSet.of(VertexState.TERMINATING, VertexState.FAILED, VertexState.KILLED, VertexState.ERROR),
+ VertexEventType.V_COMMIT_COMPLETED,
+ COMMIT_COMPLETED_TRANSITION)
// Ignore-able events
.addTransition(VertexState.TERMINATING, VertexState.TERMINATING,
EnumSet.of(VertexEventType.V_TERMINATE,
@@ -694,6 +746,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private AtomicBoolean committed = new AtomicBoolean(false);
private AtomicBoolean aborted = new AtomicBoolean(false);
+ private AtomicBoolean commitCanceled = new AtomicBoolean(false);
private boolean commitVertexOutputs = false;
private Map<String, VertexGroupInfo> dagVertexGroups;
@@ -1750,18 +1803,77 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
new DAGHistoryEvent(getDAGId(), finishEvt));
}
- static VertexState checkVertexForCompletion(final VertexImpl vertex) {
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Checking for vertex completion for "
- + vertex.logIdentifier
- + ", numTasks=" + vertex.numTasks
- + ", failedTaskCount=" + vertex.failedTaskCount
- + ", killedTaskCount=" + vertex.killedTaskCount
- + ", successfulTaskCount=" + vertex.succeededTaskCount
- + ", completedTaskCount=" + vertex.completedTaskCount
- + ", terminationCause=" + vertex.terminationCause);
+ private static VertexState commitOrFinish(final VertexImpl vertex) {
+ // commit only once. Dont commit shared outputs
+ if (vertex.outputCommitters != null
+ && !vertex.outputCommitters.isEmpty()) {
+ boolean firstCommit = true;
+ for (Entry<String, OutputCommitter> entry : vertex.outputCommitters.entrySet()) {
+ final OutputCommitter committer = entry.getValue();
+ final String outputName = entry.getKey();
+ if (vertex.sharedOutputs.contains(outputName)) {
+ // dont commit shared committers. Will be committed by the DAG
+ continue;
+ }
+ if (firstCommit) {
+ LOG.info("Invoking committer commit for vertex, vertexId="
+ + vertex.logIdentifier);
+ // Log commit start event on first actual commit
+ try {
+ vertex.appContext.getHistoryHandler().handleCriticalEvent(
+ new DAGHistoryEvent(vertex.getDAGId(),
+ new VertexCommitStartedEvent(vertex.vertexId,
+ vertex.clock.getTime())));
+ } catch (IOException e) {
+ LOG.error("Failed to persist commit start event to recovery, vertex="
+ + vertex.logIdentifier, e);
+ vertex.trySetTerminationCause(VertexTerminationCause.RECOVERY_ERROR);
+ return vertex.finished(VertexState.FAILED);
+ }
+ } else {
+ firstCommit = false;
+ }
+ VertexCommitCallback commitCallback = new VertexCommitCallback(vertex, outputName);
+ CallableEvent commitCallableEvent = new CallableEvent(commitCallback) {
+ @Override
+ public Void call() throws Exception {
+ vertex.dagUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ LOG.info("Invoking committer commit for output=" + outputName
+ + ", vertexId=" + vertex.logIdentifier);
+ committer.commitOutput();
+ return null;
+ }
+ });
+ return null;
+ }
+ };
+ ListenableFuture<Void> commitFuture =
+ vertex.getAppContext().getExecService().submit(commitCallableEvent);
+ Futures.addCallback(commitFuture, commitCallableEvent.getCallback());
+ vertex.commitFutures.put(outputName, commitFuture);
+ }
+ }
+ if (vertex.commitFutures.isEmpty()) {
+ return vertex.finished(VertexState.SUCCEEDED);
+ } else {
+ return VertexState.COMMITTING;
}
+ }
+
+ // triggered by task_complete
+ static VertexState checkTasksForCompletion(final VertexImpl vertex) {
+
+ LOG.info("Checking tasks for vertex completion for "
+ + vertex.logIdentifier
+ + ", numTasks=" + vertex.numTasks
+ + ", failedTaskCount=" + vertex.failedTaskCount
+ + ", killedTaskCount=" + vertex.killedTaskCount
+ + ", successfulTaskCount=" + vertex.succeededTaskCount
+ + ", completedTaskCount=" + vertex.completedTaskCount
+ + ", commitInProgress=" + vertex.commitFutures.size()
+ + ", terminationCause=" + vertex.terminationCause);
//check for vertex failure first
if (vertex.completedTaskCount > vertex.tasks.size()) {
@@ -1781,161 +1893,171 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
//Only succeed if tasks complete successfully and no terminationCause is registered.
if(vertex.succeededTaskCount == vertex.tasks.size() && vertex.terminationCause == null) {
- LOG.info("Vertex succeeded: " + vertex.logIdentifier);
- try {
- if (vertex.commitVertexOutputs && !vertex.committed.getAndSet(true)) {
- // commit only once. Dont commit shared outputs
- LOG.info("Invoking committer commit for vertex, vertexId="
- + vertex.logIdentifier);
- if (vertex.outputCommitters != null
- && !vertex.outputCommitters.isEmpty()) {
- boolean firstCommit = true;
- for (Entry<String, OutputCommitter> entry : vertex.outputCommitters.entrySet()) {
- final OutputCommitter committer = entry.getValue();
- final String outputName = entry.getKey();
- if (vertex.sharedOutputs.contains(outputName)) {
- // dont commit shared committers. Will be committed by the DAG
- continue;
- }
- if (firstCommit) {
- // Log commit start event on first actual commit
- try {
- vertex.appContext.getHistoryHandler().handleCriticalEvent(
- new DAGHistoryEvent(vertex.getDAGId(),
- new VertexCommitStartedEvent(vertex.vertexId,
- vertex.clock.getTime())));
- } catch (IOException e) {
- LOG.error("Failed to persist commit start event to recovery, vertex="
- + vertex.logIdentifier, e);
- vertex.trySetTerminationCause(VertexTerminationCause.INTERNAL_ERROR);
- return vertex.finished(VertexState.FAILED);
- }
- } else {
- firstCommit = false;
- }
- vertex.dagUgi.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- LOG.info("Invoking committer commit for output=" + outputName
- + ", vertexId=" + vertex.logIdentifier);
- committer.commitOutput();
- return null;
- }
- });
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Failed to do commit on vertex, vertexId="
- + vertex.logIdentifier, e);
- vertex.trySetTerminationCause(VertexTerminationCause.COMMIT_FAILURE);
- return vertex.finished(VertexState.FAILED);
- }
- return vertex.finished(VertexState.SUCCEEDED);
- }
- else if(vertex.terminationCause == VertexTerminationCause.DAG_KILL ){
- vertex.setFinishTime();
- String diagnosticMsg = "Vertex killed due to user-initiated job kill. "
- + "failedTasks:"
- + vertex.failedTaskCount;
- LOG.info(diagnosticMsg);
- vertex.addDiagnostic(diagnosticMsg);
- vertex.abortVertex(VertexStatus.State.KILLED);
- return vertex.finished(VertexState.KILLED);
- }
- else if(vertex.terminationCause == VertexTerminationCause.OTHER_VERTEX_FAILURE ){
- vertex.setFinishTime();
- String diagnosticMsg = "Vertex killed as other vertex failed. "
- + "failedTasks:"
- + vertex.failedTaskCount;
- LOG.info(diagnosticMsg);
- vertex.addDiagnostic(diagnosticMsg);
- vertex.abortVertex(VertexStatus.State.KILLED);
- return vertex.finished(VertexState.KILLED);
- }
- else if(vertex.terminationCause == VertexTerminationCause.OWN_TASK_FAILURE ){
- if(vertex.failedTaskCount == 0){
- LOG.error("task failure accounting error. terminationCause=TASK_FAILURE but vertex.failedTaskCount == 0");
+ LOG.info("All tasks are succeeded, vertex:" + vertex.logIdentifier);
+ if (vertex.commitVertexOutputs && !vertex.committed.getAndSet(true)) {
+ // start commit if there're commits or just finish if no commits
+ return commitOrFinish(vertex);
+ } else {
+ // just finish because no vertex committing needed
+ return vertex.finished(VertexState.SUCCEEDED);
}
- vertex.setFinishTime();
- String diagnosticMsg = "Vertex failed as one or more tasks failed. "
- + "failedTasks:"
- + vertex.failedTaskCount;
- LOG.info(diagnosticMsg);
- vertex.addDiagnostic(diagnosticMsg);
- vertex.abortVertex(VertexStatus.State.FAILED);
- return vertex.finished(VertexState.FAILED);
- }
- else if (vertex.terminationCause == VertexTerminationCause.INTERNAL_ERROR) {
- vertex.setFinishTime();
- String diagnosticMsg = "Vertex failed/killed due to internal error. "
- + "failedTasks:"
- + vertex.failedTaskCount
- + " killedTasks:"
- + vertex.killedTaskCount;
- LOG.info(diagnosticMsg);
- vertex.abortVertex(State.FAILED);
- return vertex.finished(VertexState.FAILED);
- }
- else if (vertex.terminationCause == VertexTerminationCause.AM_USERCODE_FAILURE) {
- vertex.setFinishTime();
- String diagnosticMsg = "Vertex failed/killed due to VertexManagerPlugin/EdgeManagerPlugin failed. "
- + "failedTasks:"
- + vertex.failedTaskCount
- + " killedTasks:"
- + vertex.killedTaskCount;
- LOG.info(diagnosticMsg);
- vertex.abortVertex(State.FAILED);
- return vertex.finished(VertexState.FAILED);
- }
- else if (vertex.terminationCause == VertexTerminationCause.ROOT_INPUT_INIT_FAILURE) {
- vertex.setFinishTime();
- String diagnosticMsg = "Vertex failed/killed due to ROOT_INPUT_INIT_FAILURE failed. "
- + "failedTasks:"
- + vertex.failedTaskCount
- + " killedTasks:"
- + vertex.killedTaskCount;
- LOG.info(diagnosticMsg);
- vertex.abortVertex(State.FAILED);
- return vertex.finished(VertexState.FAILED);
- }
- else if (vertex.terminationCause == VertexTerminationCause.COMMIT_FAILURE) {
- vertex.setFinishTime();
- String diagnosticMsg = "Vertex failed/killed due to COMMIT_FAILURE failed. "
- + "failedTasks:"
- + vertex.failedTaskCount
- + " killedTasks:"
- + vertex.killedTaskCount;
- LOG.info(diagnosticMsg);
- vertex.abortVertex(State.FAILED);
- return vertex.finished(VertexState.FAILED);
- }
- else if (vertex.terminationCause == VertexTerminationCause.VERTEX_RERUN_AFTER_COMMIT) {
- vertex.setFinishTime();
- String diagnosticMsg = "Vertex failed/killed due to invalid rerun failed. "
- + "failedTasks:"
- + vertex.failedTaskCount
- + " killedTasks:"
- + vertex.killedTaskCount;
- LOG.info(diagnosticMsg);
- vertex.abortVertex(State.FAILED);
- return vertex.finished(VertexState.FAILED);
- }
- else {
- //should never occur
- throw new TezUncheckedException("All tasks complete, but cannot determine final state of vertex:" + vertex.logIdentifier
- + ", failedTaskCount=" + vertex.failedTaskCount
- + ", killedTaskCount=" + vertex.killedTaskCount
- + ", successfulTaskCount=" + vertex.succeededTaskCount
- + ", completedTaskCount=" + vertex.completedTaskCount
- + ", terminationCause=" + vertex.terminationCause);
}
+ return finishWithTerminationCause(vertex);
}
//return the current state, Vertex not finished yet
return vertex.getInternalState();
}
+ //triggered by commit_complete
+ static VertexState checkCommitsForCompletion(final VertexImpl vertex) {
+ LOG.info("Checking commits for vertex completion for "
+ + vertex.logIdentifier
+ + ", numTasks=" + vertex.numTasks
+ + ", failedTaskCount=" + vertex.failedTaskCount
+ + ", killedTaskCount=" + vertex.killedTaskCount
+ + ", successfulTaskCount=" + vertex.succeededTaskCount
+ + ", completedTaskCount=" + vertex.completedTaskCount
+ + ", commitInProgress=" + vertex.commitFutures.size()
+ + ", terminationCause=" + vertex.terminationCause);
+
+ // terminationCause is null mean commit is succeeded, otherwise terminationCause will be set.
+ if (vertex.terminationCause == null) {
+ Preconditions.checkState(vertex.getState() == VertexState.COMMITTING,
+ "Vertex should be in COMMITTING state, but in " + vertex.getState()
+ + ", vertex:" + vertex.getLogIdentifier());
+ if (vertex.commitFutures.isEmpty()) {
+ // move from COMMITTING to SUCCEEDED
+ return vertex.finished(VertexState.SUCCEEDED);
+ } else {
+ return VertexState.COMMITTING;
+ }
+ } else {
+ if (!vertex.commitFutures.isEmpty()) {
+ // pending commits are running
+ return VertexState.TERMINATING;
+ } else {
+ // all the commits are completed successfully
+ return finishWithTerminationCause(vertex);
+ }
+ }
+ }
+
+ // TODO TEZ-2248
+ private static VertexState finishWithTerminationCause(VertexImpl vertex) {
+ if(vertex.terminationCause == VertexTerminationCause.DAG_KILL ){
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex killed due to user-initiated job kill. "
+ + "failedTasks:"
+ + vertex.failedTaskCount;
+ LOG.info(diagnosticMsg);
+ vertex.addDiagnostic(diagnosticMsg);
+ return vertex.finished(VertexState.KILLED);
+ }
+ else if(vertex.terminationCause == VertexTerminationCause.OTHER_VERTEX_FAILURE ){
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex killed as other vertex failed. "
+ + "failedTasks:"
+ + vertex.failedTaskCount;
+ LOG.info(diagnosticMsg);
+ vertex.addDiagnostic(diagnosticMsg);
+ return vertex.finished(VertexState.KILLED);
+ }
+ else if(vertex.terminationCause == VertexTerminationCause.OWN_TASK_FAILURE ){
+ if(vertex.failedTaskCount == 0){
+ LOG.error("task failure accounting error. terminationCause=TASK_FAILURE but vertex.failedTaskCount == 0");
+ }
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex failed as one or more tasks failed. "
+ + "failedTasks:"
+ + vertex.failedTaskCount;
+ LOG.info(diagnosticMsg);
+ vertex.addDiagnostic(diagnosticMsg);
+ return vertex.finished(VertexState.FAILED);
+ }
+ else if (vertex.terminationCause == VertexTerminationCause.INTERNAL_ERROR) {
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex failed/killed due to internal error. "
+ + "failedTasks:"
+ + vertex.failedTaskCount
+ + " killedTasks:"
+ + vertex.killedTaskCount;
+ LOG.info(diagnosticMsg);
+ return vertex.finished(VertexState.FAILED);
+ }
+ else if (vertex.terminationCause == VertexTerminationCause.AM_USERCODE_FAILURE) {
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex failed/killed due to VertexManagerPlugin/EdgeManagerPlugin failed. "
+ + "failedTasks:"
+ + vertex.failedTaskCount
+ + " killedTasks:"
+ + vertex.killedTaskCount;
+ LOG.info(diagnosticMsg);
+ return vertex.finished(VertexState.FAILED);
+ }
+ else if (vertex.terminationCause == VertexTerminationCause.ROOT_INPUT_INIT_FAILURE) {
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex failed/killed due to ROOT_INPUT_INIT_FAILURE failed. "
+ + "failedTasks:"
+ + vertex.failedTaskCount
+ + " killedTasks:"
+ + vertex.killedTaskCount;
+ LOG.info(diagnosticMsg);
+ return vertex.finished(VertexState.FAILED);
+ }
+ else if (vertex.terminationCause == VertexTerminationCause.COMMIT_FAILURE) {
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex failed/killed due to COMMIT_FAILURE failed. "
+ + "failedTasks:"
+ + vertex.failedTaskCount
+ + " killedTasks:"
+ + vertex.killedTaskCount;
+ LOG.info(diagnosticMsg);
+ return vertex.finished(VertexState.FAILED);
+ }
+ else if (vertex.terminationCause == VertexTerminationCause.VERTEX_RERUN_AFTER_COMMIT) {
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex failed/killed due to vertex-rerun after commit. "
+ + "failedTasks:"
+ + vertex.failedTaskCount
+ + " killedTasks:"
+ + vertex.killedTaskCount;
+ LOG.info(diagnosticMsg);
+ return vertex.finished(VertexState.FAILED);
+ }
+ else if (vertex.terminationCause == VertexTerminationCause.VERTEX_RERUN_IN_COMMITTING) {
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex failed/killed due to vertex-rerun in commiting. "
+ + "failedTasks:"
+ + vertex.failedTaskCount
+ + " killedTasks:"
+ + vertex.killedTaskCount;
+ LOG.info(diagnosticMsg);
+ return vertex.finished(VertexState.FAILED);
+ }
+ else if (vertex.terminationCause == VertexTerminationCause.RECOVERY_ERROR) {
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex failed/killed due to recovery error. "
+ + "failedTasks:"
+ + vertex.failedTaskCount
+ + " killedTasks:"
+ + vertex.killedTaskCount;
+ LOG.info(diagnosticMsg);
+ return vertex.finished(VertexState.FAILED);
+ }
+ else {
+ //should never occur
+ throw new TezUncheckedException("All tasks & commits complete, but cannot determine final state of vertex:"
+ + vertex.logIdentifier
+ + ", failedTaskCount=" + vertex.failedTaskCount
+ + ", killedTaskCount=" + vertex.killedTaskCount
+ + ", successfulTaskCount=" + vertex.succeededTaskCount
+ + ", completedTaskCount=" + vertex.completedTaskCount
+ + ", commitInProgress=" + vertex.commitFutures.size()
+ + ", terminationCause=" + vertex.terminationCause);
+ }
+ }
+
/**
* Set the terminationCause and send a kill-message to all tasks.
* The task-kill messages are only sent once.
@@ -1974,6 +2096,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if (!StringUtils.isEmpty(diag)) {
addDiagnostic(diag);
}
+ abortVertex(VertexStatus.State.valueOf(finalState.name()));
eventHandler.handle(new DAGEvent(getDAGId(),
DAGEventType.INTERNAL_ERROR));
try {
@@ -1988,6 +2111,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if (!StringUtils.isEmpty(diag)) {
addDiagnostic(diag);
}
+ abortVertex(VertexStatus.State.valueOf(finalState.name()));
eventHandler.handle(new DAGEventVertexCompleted(getVertexId(),
finalState, terminationCause));
try {
@@ -2074,7 +2198,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
addDiagnostic("Vertex init failed : "
+ ExceptionUtils.getStackTrace(e));
trySetTerminationCause(VertexTerminationCause.INIT_FAILURE);
- abortVertex(VertexStatus.State.FAILED);
finished(VertexState.FAILED);
return false;
}
@@ -2289,7 +2412,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
+ ", numTasks=" + numTasks);
trySetTerminationCause(VertexTerminationCause.INVALID_NUM_OF_TASKS);
if (event != null) {
- abortVertex(VertexStatus.State.FAILED);
return finished(VertexState.FAILED);
} else {
return VertexState.FAILED;
@@ -3441,13 +3563,14 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
return VertexState.RUNNING;
}
- private void abortVertex(final VertexStatus.State finalState) {
+ void abortVertex(final VertexStatus.State finalState) {
if (this.aborted.getAndSet(true)) {
LOG.info("Ignoring multiple aborts for vertex: " + logIdentifier);
return;
}
- LOG.info("Invoking committer abort for vertex, vertexId=" + logIdentifier);
+
if (outputCommitters != null) {
+ LOG.info("Invoking committer abort for vertex, vertexId=" + logIdentifier);
try {
dagUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
@@ -3552,7 +3675,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
public void transition(VertexImpl vertex, VertexEvent event) {
VertexEventTermination vet = (VertexEventTermination) event;
vertex.trySetTerminationCause(vet.getTerminationCause());
- vertex.abortVertex(VertexStatus.State.KILLED);
vertex.addDiagnostic("Vertex received Kill in INITED state.");
vertex.finished(VertexState.KILLED);
}
@@ -3592,6 +3714,24 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
}
+ private static class VertexKilledWhileCommittingTransition
+ implements SingleArcTransition<VertexImpl, VertexEvent> {
+
+ @Override
+ public void transition(VertexImpl vertex, VertexEvent event) {
+
+
+ VertexEventTermination vet = (VertexEventTermination) event;
+ VertexTerminationCause trigger = vet.getTerminationCause();
+ String msg = "Vertex received Kill while in COMMITTING state, terminationCause="
+ + trigger +", vertex=" + vertex.logIdentifier;
+ LOG.info(msg);
+ vertex.addDiagnostic(msg);
+ vertex.trySetTerminationCause(trigger);
+ vertex.cancelCommits();
+ }
+ }
+
private static class VertexManagerUserCodeErrorTransition implements
MultipleArcTransition<VertexImpl, VertexEvent, VertexState> {
@Override
@@ -3608,10 +3748,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
vertex.terminationCause = VertexTerminationCause.AM_USERCODE_FAILURE;
vertex.recoveredState = VertexState.FAILED;
return VertexState.RECOVERING;
- } else if (vertex.getState() == VertexState.RUNNING) {
+ } else if (vertex.getState() == VertexState.RUNNING || vertex.getState() == VertexState.COMMITTING) {
vertex.addDiagnostic(msg + "," + ExceptionUtils.getStackTrace(e.getCause()));
vertex.tryEnactKill(VertexTerminationCause.AM_USERCODE_FAILURE,
TaskTerminationCause.AM_USERCODE_FAILURE);
+ vertex.cancelCommits();
return VertexState.TERMINATING;
} else {
vertex.finished(VertexState.FAILED,
@@ -3707,7 +3848,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
taskKilled(vertex, task);
}
- VertexState state = VertexImpl.checkVertexForCompletion(vertex);
+ VertexState state = VertexImpl.checkTasksForCompletion(vertex);
if(state == VertexState.RUNNING && forceTransitionToKillWait){
return VertexState.TERMINATING;
}
@@ -3752,7 +3893,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@Override
public VertexState transition(VertexImpl vertex, VertexEvent event) {
- return VertexImpl.checkVertexForCompletion(vertex);
+ return VertexImpl.checkTasksForCompletion(vertex);
}
}
@@ -3762,26 +3903,38 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
public VertexState transition(VertexImpl vertex, VertexEvent event) {
VertexEventTaskCompleted vEvent = (VertexEventTaskCompleted) event;
VertexState finalState;
- VertexStatus.State finalStatus;
String diagnosticMsg;
if (vEvent.getState() == TaskState.FAILED) {
finalState = VertexState.FAILED;
- finalStatus = VertexStatus.State.FAILED;
diagnosticMsg = "Vertex " + vertex.logIdentifier +" failed as task " + vEvent.getTaskID() +
" failed after vertex succeeded.";
} else {
finalState = VertexState.ERROR;
- finalStatus = VertexStatus.State.ERROR;
diagnosticMsg = "Vertex " + vertex.logIdentifier + " error as task " + vEvent.getTaskID() +
" completed with state " + vEvent.getState() + " after vertex succeeded.";
}
LOG.info(diagnosticMsg);
- vertex.abortVertex(finalStatus);
vertex.finished(finalState, VertexTerminationCause.OWN_TASK_FAILURE, diagnosticMsg);
return finalState;
}
}
+ private static class TaskRescheduledWhileCommittingTransition implements
+ SingleArcTransition<VertexImpl, VertexEvent> {
+
+ @Override
+ public void transition(VertexImpl vertex, VertexEvent event) {
+ // terminate any running tasks
+ String diagnosticMsg = vertex.getLogIdentifier() + " failed due to in-committing rescheduling of "
+ + ((VertexEventTaskReschedule)event).getTaskID();
+ LOG.info(diagnosticMsg);
+ vertex.addDiagnostic(diagnosticMsg);
+ vertex.tryEnactKill(VertexTerminationCause.VERTEX_RERUN_IN_COMMITTING,
+ TaskTerminationCause.TASK_RESCHEDULE_IN_COMMITTING);
+ vertex.cancelCommits();
+ }
+ }
+
private static class TaskRescheduledAfterVertexSuccessTransition implements
MultipleArcTransition<VertexImpl, VertexEvent, VertexState> {
@@ -3806,12 +3959,47 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
LOG.info(diagnosticMsg);
vertex.tryEnactKill(VertexTerminationCause.OWN_TASK_FAILURE,
TaskTerminationCause.OWN_TASK_FAILURE);
- vertex.abortVertex(VertexStatus.State.FAILED);
vertex.finished(VertexState.FAILED, VertexTerminationCause.OWN_TASK_FAILURE, diagnosticMsg);
return VertexState.FAILED;
}
}
+ private void commitCompleted(VertexEventCommitCompleted commitCompletedEvent) {
+ Preconditions.checkState(commitFutures.remove(commitCompletedEvent.getOutputName()) != null,
+ "Unknown commit:" + commitCompletedEvent.getOutputName() + ", vertex=" + logIdentifier);
+ if (commitCompletedEvent.isSucceeded()) {
+ LOG.info("Commit succeeded for output:" + commitCompletedEvent.getOutputName()
+ + ", vertexId=" + logIdentifier);
+ } else {
+ String diag = "Commit failed for output:" + commitCompletedEvent.getOutputName()
+ + ", vertexId=" + logIdentifier + ", "
+ + ExceptionUtils.getStackTrace(commitCompletedEvent.getException());;
+ LOG.info(diag);
+ addDiagnostic(diag);
+ trySetTerminationCause(VertexTerminationCause.COMMIT_FAILURE);
+ cancelCommits();
+ }
+ }
+
+ private static class CommitCompletedTransition implements
+ MultipleArcTransition<VertexImpl, VertexEvent, VertexState> {
+
+ @Override
+ public VertexState transition(VertexImpl vertex, VertexEvent event) {
+ vertex.commitCompleted((VertexEventCommitCompleted)event);
+ return checkCommitsForCompletion(vertex);
+ }
+ }
+
+ private void cancelCommits() {
+ if (!this.commitCanceled.getAndSet(true)) {
+ for (Map.Entry<String, ListenableFuture<Void>> entry : commitFutures.entrySet()) {
+ LOG.info("Canceling commit of output:" + entry.getKey() + ", vertexId=" + logIdentifier);
+ entry.getValue().cancel(true);
+ }
+ }
+ }
+
private void addDiagnostic(String diag) {
diagnostics.add(diag);
}
@@ -3853,9 +4041,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
} catch (AMUserCodeException e) {
String msg = "Exception in " + e.getSource() + ", vertex=" + vertex.getLogIdentifier();
LOG.error(msg, e);
- if (vertex.getState() == VertexState.RUNNING) {
+ if (vertex.getState() == VertexState.RUNNING || vertex.getState() == VertexState.COMMITTING) {
vertex.addDiagnostic(msg + ", " + e.getMessage() + ", " + ExceptionUtils.getStackTrace(e.getCause()));
vertex.tryEnactKill(VertexTerminationCause.AM_USERCODE_FAILURE, TaskTerminationCause.AM_USERCODE_FAILURE);
+ vertex.cancelCommits();
return VertexState.TERMINATING;
} else {
vertex.finished(VertexState.FAILED, VertexTerminationCause.AM_USERCODE_FAILURE,
@@ -4073,6 +4262,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
vertex.getDAGId(), "Invalid event " + event.getType()
+ " on Vertex " + vertex.getLogIdentifier()));
vertex.setFinishTime();
+ vertex.trySetTerminationCause(VertexTerminationCause.INTERNAL_ERROR);
+ vertex.cancelCommits();
vertex.finished(VertexState.ERROR);
}
}
@@ -4133,6 +4324,30 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
}
+ private static class VertexCommitCallback implements FutureCallback<Void>{
+
+ private String outputName;
+ private VertexImpl vertex;
+
+ public VertexCommitCallback(VertexImpl vertex, String outputName) {
+ this.vertex = vertex;
+ this.outputName = outputName;
+ }
+
+ @Override
+ public void onSuccess(Void result) {
+ vertex.getEventHandler().handle(
+ new VertexEventCommitCompleted(vertex.vertexId, outputName, true, null));
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ vertex.getEventHandler().handle(
+ new VertexEventCommitCompleted(vertex.vertexId, outputName, false, t));
+ }
+
+ }
+
@Override
public void setInputVertices(Map<Vertex, Edge> inVertices) {
this.sourceVertices = inVertices;
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
index 8be60c5..4d01117 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
@@ -28,6 +28,9 @@ import java.util.Locale;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -49,21 +52,26 @@ import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
import org.apache.tez.dag.api.DataSinkDescriptor;
+import org.apache.tez.dag.api.GroupInputEdge;
import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.OutputCommitterDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.VertexGroup;
import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.client.VertexStatus.State;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.MockDAGAppMaster.CountersDelegate;
import org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher;
import org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher.ContainerData;
import org.apache.tez.dag.app.MockDAGAppMaster.StatisticsDelegate;
import org.apache.tez.dag.app.dag.DAGState;
-import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.app.dag.event.DAGAppMasterEventSchedulingServiceError;
import org.apache.tez.dag.app.dag.impl.DAGImpl;
@@ -73,6 +81,8 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.runtime.api.VertexStatistics;
+import org.apache.tez.runtime.api.OutputCommitter;
+import org.apache.tez.runtime.api.OutputCommitterContext;
import org.apache.tez.runtime.api.events.DataMovementEvent;
import org.apache.tez.runtime.api.impl.IOStatistics;
import org.apache.tez.runtime.api.impl.InputSpec;
@@ -85,8 +95,10 @@ import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Maps;
+import com.google.common.primitives.Ints;
public class TestMockDAGAppMaster {
+ private static final Log LOG = LogFactory.getLog(TestMockDAGAppMaster.class);
static Configuration defaultConf;
static FileSystem localFs;
@@ -172,7 +184,7 @@ public class TestMockDAGAppMaster {
Assert.assertEquals(TaskAttemptState.KILLED, killedTa.getState());
tezClient.stop();
}
-
+
@Test (timeout = 5000)
public void testBasicEvents() throws Exception {
TezConfiguration tezconf = new TezConfiguration(defaultConf);
@@ -647,4 +659,224 @@ public class TestMockDAGAppMaster {
mockApp.waitForServiceToStop(Integer.MAX_VALUE);
}
}
+
+
+ private OutputCommitterDescriptor createOutputCommitterDesc(boolean failOnCommit) {
+ OutputCommitterDescriptor outputCommitterDesc =
+ OutputCommitterDescriptor.create(FailingOutputCommitter.class.getName());
+ UserPayload payload = UserPayload.create(
+ ByteBuffer.wrap(new FailingOutputCommitter.FailingOutputCommitterConfig(failOnCommit).toUserPayload()));
+ outputCommitterDesc.setUserPayload(payload);
+ return outputCommitterDesc;
+ }
+
+ private DAG createDAG(String dagName, boolean uv12CommitFail, boolean v3CommitFail) {
+ DAG dag = DAG.create(dagName);
+ Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("Proc"), 1);
+ Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("Proc"), 1);
+ Vertex v3 = Vertex.create("v3", ProcessorDescriptor.create("Proc"), 1);
+ VertexGroup uv12 = dag.createVertexGroup("uv12", v1, v2);
+ DataSinkDescriptor uv12DataSink = DataSinkDescriptor.create(
+ OutputDescriptor.create("dummy output"), createOutputCommitterDesc(uv12CommitFail), null);
+ uv12.addDataSink("uv12Out", uv12DataSink);
+ DataSinkDescriptor v3DataSink = DataSinkDescriptor.create(
+ OutputDescriptor.create("dummy output"), createOutputCommitterDesc(v3CommitFail), null);
+ v3.addDataSink("v3Out", v3DataSink);
+
+ GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(
+ DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")), InputDescriptor
+ .create("merge.class"));
+ dag.addVertex(v1)
+ .addVertex(v2)
+ .addVertex(v3)
+ .addEdge(e1);
+ return dag;
+ }
+
+ @Test (timeout = 60000)
+ public void testCommitOutputOnDAGSuccess() throws Exception {
+ TezConfiguration tezconf = new TezConfiguration(defaultConf);
+ MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
+ tezClient.start();
+
+ // both committers succeed
+ DAG dag1 = createDAG("testDAGBothCommitsSucceed", false, false);
+ DAGClient dagClient = tezClient.submitDAG(dag1);
+ dagClient.waitForCompletion();
+ Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
+
+ // vertexGroupCommiter fail (uv12)
+ DAG dag2 = createDAG("testDAGVertexGroupCommitFail", true, false);
+ dagClient = tezClient.submitDAG(dag2);
+ dagClient.waitForCompletion();
+ Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
+ LOG.info(dagClient.getDAGStatus(null).getDiagnostics());
+ Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),",")
+ .contains("fail output committer:uv12Out"));
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState());
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v2", null).getState());
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v3", null).getState());
+
+ // vertex commit fail (v3)
+ DAG dag3 = createDAG("testDAGVertexCommitFail", false, true);
+ dagClient = tezClient.submitDAG(dag3);
+ dagClient.waitForCompletion();
+ LOG.info(dagClient.getDAGStatus(null).getDiagnostics());
+ Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
+ Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),",")
+ .contains("fail output committer:v3Out"));
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState());
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v2", null).getState());
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v3", null).getState());
+
+ // both committers fail
+ DAG dag4 = createDAG("testDAGBothCommitsFail", true, true);
+ dagClient = tezClient.submitDAG(dag4);
+ dagClient.waitForCompletion();
+ LOG.info(dagClient.getDAGStatus(null).getDiagnostics());
+ Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
+ String diag = StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),",");
+ Assert.assertTrue(diag.contains("fail output committer:uv12Out") ||
+ diag.contains("fail output committer:v3Out"));
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState());
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v2", null).getState());
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v3", null).getState());
+
+ tezClient.stop();
+ }
+
+ @Test (timeout = 60000)
+ public void testCommitOutputOnVertexSuccess() throws Exception {
+ TezConfiguration tezconf = new TezConfiguration(defaultConf);
+ tezconf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
+ MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
+ tezClient.start();
+
+ // both committers succeed
+ DAG dag1 = createDAG("testDAGBothCommitsSucceed", false, false);
+ DAGClient dagClient = tezClient.submitDAG(dag1);
+ dagClient.waitForCompletion();
+ Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
+
+ // vertexGroupCommiter fail (uv12)
+ DAG dag2 = createDAG("testDAGVertexGroupCommitFail", true, false);
+ dagClient = tezClient.submitDAG(dag2);
+ dagClient.waitForCompletion();
+ Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
+ LOG.info(dagClient.getDAGStatus(null).getDiagnostics());
+ Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),",")
+ .contains("fail output committer:uv12Out"));
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState());
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v2", null).getState());
+ VertexStatus.State v3State = dagClient.getVertexStatus("v3", null).getState();
+ // v3 either succeeded (commit completed before uv12 commit fails)
+ // or killed ( uv12 commit fail when v3 is in running/committing)
+ if (v3State.equals(VertexStatus.State.SUCCEEDED)) {
+ LOG.info("v3 is succeeded");
+ } else {
+ Assert.assertEquals(VertexStatus.State.KILLED, v3State);
+ }
+
+ // vertex commit fail (v3)
+ DAG dag3 = createDAG("testDAGVertexCommitFail", false, true);
+ dagClient = tezClient.submitDAG(dag3);
+ dagClient.waitForCompletion();
+ LOG.info(dagClient.getDAGStatus(null).getDiagnostics());
+ Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
+ Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),",")
+ .contains("Commit failed"));
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState());
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v2", null).getState());
+ Assert.assertEquals(VertexStatus.State.FAILED, dagClient.getVertexStatus("v3", null).getState());
+ Assert.assertTrue(StringUtils.join(dagClient.getVertexStatus("v3", null).getDiagnostics(),",")
+ .contains("fail output committer:v3Out"));
+
+ // both committers fail
+ DAG dag4 = createDAG("testDAGBothCommitsFail", true, true);
+ dagClient = tezClient.submitDAG(dag4);
+ dagClient.waitForCompletion();
+ Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
+ LOG.info(dagClient.getDAGStatus(null).getDiagnostics());
+ Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
+ String diag = StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),",");
+ Assert.assertTrue(diag.contains("fail output committer:uv12Out") ||
+ diag.contains("fail output committer:v3Out"));
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState());
+ Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v2", null).getState());
+ v3State = dagClient.getVertexStatus("v3", null).getState();
+ // v3 either failed (commit of v3 fail before uv12 commit)
+ // or killed ( uv12 commit fail before commit of v3)
+ if (v3State.equals(VertexStatus.State.FAILED)) {
+ LOG.info("v3 is failed");
+ Assert.assertTrue(StringUtils.join(dagClient.getVertexStatus("v3", null).getDiagnostics(),",")
+ .contains("fail output committer:v3Out"));
+ } else {
+ Assert.assertEquals(VertexStatus.State.KILLED, v3State);
+ }
+
+ tezClient.stop();
+ }
+
+ public static class FailingOutputCommitter extends OutputCommitter {
+
+ boolean failOnCommit = false;
+
+ public FailingOutputCommitter(OutputCommitterContext committerContext) {
+ super(committerContext);
+ }
+
+ @Override
+ public void initialize() throws Exception {
+ FailingOutputCommitterConfig config = new
+ FailingOutputCommitterConfig();
+ config.fromUserPayload(
+ getContext().getUserPayload().deepCopyAsArray());
+ failOnCommit = config.failOnCommit;
+ }
+
+ @Override
+ public void setupOutput() throws Exception {
+
+ }
+
+ @Override
+ public void commitOutput() throws Exception {
+ if (failOnCommit) {
+ throw new Exception("fail output committer:" + getContext().getOutputName());
+ }
+ }
+
+ @Override
+ public void abortOutput(State finalState) throws Exception {
+
+ }
+
+ public static class FailingOutputCommitterConfig {
+ boolean failOnCommit;
+
+ public FailingOutputCommitterConfig() {
+ this(false);
+ }
+
+ public FailingOutputCommitterConfig(boolean failOnCommit) {
+ this.failOnCommit = failOnCommit;
+ }
+
+ public byte[] toUserPayload() {
+ return Ints.toByteArray((failOnCommit ? 1 : 0));
+ }
+
+ public void fromUserPayload(byte[] userPayload) {
+ int failInt = Ints.fromByteArray(userPayload);
+ if (failInt == 0) {
+ failOnCommit = false;
+ } else {
+ failOnCommit = true;
+ }
+ }
+ }
+ }
}
[22/50] [abbrv] tez git commit: TEZ-2308. Add set/get of record
counts in task/vertex statistics (bikas)
Posted by ss...@apache.org.
TEZ-2308. Add set/get of record counts in task/vertex statistics (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f46997a7
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f46997a7
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f46997a7
Branch: refs/heads/TEZ-2003
Commit: f46997a7cbc5474e3368c852f31a95c97da3b6a4
Parents: 87aac12
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Apr 21 14:01:59 2015 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Apr 21 14:01:59 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/runtime/api/InputStatistics.java | 9 +++++
.../runtime/api/InputStatisticsReporter.java | 10 ++++++
.../tez/runtime/api/OutputStatistics.java | 9 +++++
.../runtime/api/OutputStatisticsReporter.java | 9 +++++
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 17 +++++-----
.../tez/dag/app/TestMockDAGAppMaster.java | 12 +++++--
.../org/apache/tez/mapreduce/input/MRInput.java | 4 +++
.../tez/mapreduce/input/MultiMRInput.java | 5 +++
.../apache/tez/mapreduce/output/MROutput.java | 4 +++
.../tez/runtime/api/impl/IOStatistics.java | 35 +++++++++++++++++++-
.../tez/runtime/api/impl/TaskStatistics.java | 4 +--
.../runtime/api/impl/TezInputContextImpl.java | 7 ++++
.../runtime/api/impl/TezOutputContextImpl.java | 7 ++++
.../library/input/OrderedGroupedKVInput.java | 7 ++--
.../runtime/library/input/UnorderedKVInput.java | 7 ++--
.../output/OrderedPartitionedKVOutput.java | 5 ++-
.../library/output/UnorderedKVOutput.java | 6 ++--
.../output/UnorderedPartitionedKVOutput.java | 5 ++-
.../library/output/TestOnFileSortedOutput.java | 15 +++++++--
.../output/TestOnFileUnorderedKVOutput.java | 2 ++
21 files changed, 157 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4ba2f2f..5293120 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2308. Add set/get of record counts in task/vertex statistics
TEZ-2344. Tez UI: Equip basic-ember-table's cell level loading for all use cases in all DAGs table
TEZ-2313. Regression in handling obsolete events in ShuffleScheduler.
TEZ-2212. Notify components on DAG completion.
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java
index fb99f2d..1066dbb 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java
@@ -37,4 +37,13 @@ public interface InputStatistics {
* @return Data size in bytes
*/
public long getDataSize();
+
+ /**
+ * Get the numbers of items processed. These could be key-value pairs, table
+ * records etc.
+ *
+ * @return Number of items processed
+ */
+ public long getItemsProcessed();
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java
index 68a56e7..a85d25b 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java
@@ -34,4 +34,14 @@ public interface InputStatisticsReporter {
*/
public void reportDataSize(long size);
+ /**
+ * Report the number of items processed. These could be key-value pairs, table
+ * records etc.
+ *
+ * @param items
+ * number of items
+ */
+ public void reportItemsProcessed(long items);
+
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java
index 0373606..2f18a03 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java
@@ -38,4 +38,13 @@ public interface OutputStatistics {
* @return Data size in bytes
*/
public long getDataSize();
+
+ /**
+ * Get the numbers of items processed. These could be key-value pairs, table
+ * records etc.
+ *
+ * @return Number of items processed
+ */
+ public long getItemsProcessed();
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java
index fc9f1b7..1931e5c 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java
@@ -34,4 +34,13 @@ public interface OutputStatisticsReporter {
*/
public void reportDataSize(long size);
+ /**
+ * Report the number of items processed. These could be key-value pairs, table
+ * records etc.
+ *
+ * @param items
+ * number of items
+ */
+ public void reportItemsProcessed(long items);
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index b4b8062..5dfcb8e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -788,10 +788,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
return super.getDataSize();
}
- void mergeFrom(org.apache.tez.runtime.api.impl.IOStatistics other) {
- this.setDataSize(this.getDataSize() + other.getDataSize());
+ @Override
+ public long getItemsProcessed() {
+ return super.getItemsProcessed();
}
-
}
class VertexStatisticsImpl implements VertexStatistics {
@@ -813,12 +813,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
return;
}
- for (Map.Entry<String, org.apache.tez.runtime.api.impl.IOStatistics> entry : taskStats.getIOStatistics().entrySet()) {
- String edgeName = entry.getKey();
- IOStatisticsImpl myEdgeStat = ioStats.get(edgeName);
- Preconditions.checkState(myEdgeStat != null, "Unexpected IO name: " + edgeName
+ for (Map.Entry<String, org.apache.tez.runtime.api.impl.IOStatistics> entry : taskStats
+ .getIOStatistics().entrySet()) {
+ String ioName = entry.getKey();
+ IOStatisticsImpl myIOStat = ioStats.get(ioName);
+ Preconditions.checkState(myIOStat != null, "Unexpected IO name: " + ioName
+ " for vertex:" + getLogIdentifier());
- myEdgeStat.mergeFrom(entry.getValue());
+ myIOStat.mergeFrom(entry.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
index 4d01117..db1d632 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
@@ -376,6 +376,7 @@ public class TestMockDAGAppMaster {
OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
IOStatistics ioStats = new IOStatistics();
ioStats.setDataSize(1);
+ ioStats.setItemsProcessed(1);
TaskStatistics vAStats = new TaskStatistics();
vAStats.addIO(vBName, ioStats);
vAStats.addIO(sourceName, ioStats);
@@ -426,10 +427,14 @@ public class TestMockDAGAppMaster {
VertexStatistics vStats = v.getStatistics();
if (v.getName().equals(vAName)) {
Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getDataSize());
- Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getDataSize());
+ Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getDataSize());
+ Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getItemsProcessed());
+ Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getItemsProcessed());
} else {
Assert.assertEquals(2, vStats.getInputStatistics(vAName).getDataSize());
- Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getDataSize());
+ Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getDataSize());
+ Assert.assertEquals(2, vStats.getInputStatistics(vAName).getItemsProcessed());
+ Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getItemsProcessed());
}
}
@@ -521,6 +526,7 @@ public class TestMockDAGAppMaster {
IOStatistics ioStats = new IOStatistics();
ioStats.setDataSize(1);
+ ioStats.setItemsProcessed(1);
TaskStatistics vAStats = new TaskStatistics();
DAG dag = DAG.create("testBasisStatistics");
@@ -566,6 +572,8 @@ public class TestMockDAGAppMaster {
Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
Assert.assertEquals(numTasks,
dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0+vAName).getDataSize());
+ Assert.assertEquals(numTasks,
+ dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0+vAName).getItemsProcessed());
checkMemory(dag.getName(), mockApp);
tezClient.stop();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
index 2fb1647..991f6d1 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
@@ -532,6 +532,10 @@ public class MRInput extends MRInputBase {
@Override
public List<Event> close() throws IOException {
mrReader.close();
+ long inputRecords = getContext().getCounters()
+ .findCounter(TaskCounter.INPUT_RECORDS_PROCESSED).getValue();
+ getContext().getStatisticsReporter().reportItemsProcessed(inputRecords);
+
return null;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
index 9596f07..425d737 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.tez.common.counters.TaskCounter;
import org.apache.tez.mapreduce.input.base.MRInputBase;
import org.apache.tez.mapreduce.lib.MRInputUtils;
import org.apache.tez.mapreduce.lib.MRReader;
@@ -187,6 +188,10 @@ public class MultiMRInput extends MRInputBase {
for (MRReader reader : readers) {
reader.close();
}
+ long inputRecords = getContext().getCounters()
+ .findCounter(TaskCounter.INPUT_RECORDS_PROCESSED).getValue();
+ getContext().getStatisticsReporter().reportItemsProcessed(inputRecords);
+
return null;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
index 349a894..483c92b 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
@@ -507,6 +507,10 @@ public class MROutput extends AbstractLogicalOutput {
@Override
public synchronized List<Event> close() throws IOException {
flush();
+ long outputRecords = getContext().getCounters()
+ .findCounter(TaskCounter.OUTPUT_RECORDS).getValue();
+ getContext().getStatisticsReporter().reportItemsProcessed(outputRecords);
+
return null;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/IOStatistics.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/IOStatistics.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/IOStatistics.java
index ede9205..0f8b589 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/IOStatistics.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/IOStatistics.java
@@ -17,8 +17,15 @@
package org.apache.tez.runtime.api.impl;
-public class IOStatistics {
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+public class IOStatistics implements Writable {
private long dataSize = 0;
+ private long numItems = 0;
public void setDataSize(long size) {
this.dataSize = size;
@@ -27,4 +34,30 @@ public class IOStatistics {
public long getDataSize() {
return dataSize;
}
+
+ public void setItemsProcessed(long items) {
+ this.numItems = items;
+ }
+
+ public long getItemsProcessed() {
+ return numItems;
+ }
+
+ public void mergeFrom(org.apache.tez.runtime.api.impl.IOStatistics other) {
+ this.setDataSize(this.getDataSize() + other.getDataSize());
+ this.setItemsProcessed(this.getItemsProcessed() + other.getItemsProcessed());
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeLong(getDataSize());
+ out.writeLong(getItemsProcessed());
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ setDataSize(in.readLong());
+ setItemsProcessed(in.readLong());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskStatistics.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskStatistics.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskStatistics.java
index b50d099..0b4bef8 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskStatistics.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskStatistics.java
@@ -54,7 +54,7 @@ public class TaskStatistics implements Writable {
for (Map.Entry<String, IOStatistics> entry : ioStatistics.entrySet()) {
IOStatistics edgeStats = entry.getValue();
Text.writeString(out, entry.getKey());
- out.writeLong(edgeStats.getDataSize());
+ edgeStats.write(out);
}
}
@@ -64,7 +64,7 @@ public class TaskStatistics implements Writable {
for (int i=0; i<numEntries; ++i) {
String edgeName = Text.readString(in);
IOStatistics edgeStats = new IOStatistics();
- edgeStats.setDataSize(in.readLong());
+ edgeStats.readFields(in);
addIO(edgeName, edgeStats);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
index a9b7eab..f6330f3 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
@@ -66,6 +66,13 @@ public class TezInputContextImpl extends TezTaskContextImpl
runtimeTask.getTaskStatistics().getIOStatistics().get(sourceVertexName)
.setDataSize(size);
}
+
+ @Override
+ public void reportItemsProcessed(long items) {
+ // this is a concurrent map. Plus we are not adding/deleting entries
+ runtimeTask.getTaskStatistics().getIOStatistics().get(sourceVertexName)
+ .setItemsProcessed(items);
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
index 17513dd..4045113 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
@@ -62,6 +62,13 @@ public class TezOutputContextImpl extends TezTaskContextImpl
runtimeTask.getTaskStatistics().getIOStatistics().get(destinationVertexName)
.setDataSize(size);
}
+
+ @Override
+ public void reportItemsProcessed(long items) {
+ // this is a concurrent map. Plus we are not adding/deleting entries
+ runtimeTask.getTaskStatistics().getIOStatistics().get(destinationVertexName)
+ .setItemsProcessed(items);;
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java
index d2d9ed8..e61dbdc 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java
@@ -186,9 +186,12 @@ public class OrderedGroupedKVInput extends AbstractLogicalInput {
shuffle.shutdown();
}
- long outputSize = getContext().getCounters()
+ long dataSize = getContext().getCounters()
.findCounter(TaskCounter.SHUFFLE_BYTES_DECOMPRESSED).getValue();
- getContext().getStatisticsReporter().reportDataSize(outputSize);
+ getContext().getStatisticsReporter().reportDataSize(dataSize);
+ long inputRecords = getContext().getCounters()
+ .findCounter(TaskCounter.REDUCE_INPUT_RECORDS).getValue();
+ getContext().getStatisticsReporter().reportItemsProcessed(inputRecords);
return Collections.emptyList();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
index f2f47e1..ce27103 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
@@ -209,9 +209,12 @@ public class UnorderedKVInput extends AbstractLogicalInput {
this.shuffleManager.shutdown();
}
- long outputSize = getContext().getCounters()
+ long dataSize = getContext().getCounters()
.findCounter(TaskCounter.SHUFFLE_BYTES_DECOMPRESSED).getValue();
- getContext().getStatisticsReporter().reportDataSize(outputSize);
+ getContext().getStatisticsReporter().reportDataSize(dataSize);
+ long inputRecords = getContext().getCounters()
+ .findCounter(TaskCounter.INPUT_RECORDS_PROCESSED).getValue();
+ getContext().getStatisticsReporter().reportItemsProcessed(inputRecords);
return null;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
index df6daf2..40edc76 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
@@ -192,7 +192,10 @@ public class OrderedPartitionedKVOutput extends AbstractLogicalOutput {
long outputSize = getContext().getCounters().findCounter(TaskCounter.OUTPUT_BYTES).getValue();
getContext().getStatisticsReporter().reportDataSize(outputSize);
-
+ long outputRecords = getContext().getCounters()
+ .findCounter(TaskCounter.OUTPUT_RECORDS).getValue();
+ getContext().getStatisticsReporter().reportItemsProcessed(outputRecords);
+
return returnEvents;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
index 6c9077e..2c26374 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
@@ -44,7 +44,6 @@ import org.apache.tez.runtime.library.api.Partitioner;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler;
import org.apache.tez.runtime.library.common.writers.UnorderedPartitionedKVWriter;
-import org.apache.tez.runtime.library.partitioner.HashPartitioner;
import com.google.common.annotations.VisibleForTesting;
@@ -131,7 +130,10 @@ public class UnorderedKVOutput extends AbstractLogicalOutput {
long outputSize = getContext().getCounters().findCounter(TaskCounter.OUTPUT_BYTES).getValue();
getContext().getStatisticsReporter().reportDataSize(outputSize);
-
+ long outputRecords = getContext().getCounters()
+ .findCounter(TaskCounter.OUTPUT_RECORDS).getValue();
+ getContext().getStatisticsReporter().reportItemsProcessed(outputRecords);
+
return returnEvents;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
index 8b83f9b..34f2e3e 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
@@ -107,7 +107,10 @@ public class UnorderedPartitionedKVOutput extends AbstractLogicalOutput {
long outputSize = getContext().getCounters().findCounter(TaskCounter.OUTPUT_BYTES).getValue();
getContext().getStatisticsReporter().reportDataSize(outputSize);
-
+ long outputRecords = getContext().getCounters()
+ .findCounter(TaskCounter.OUTPUT_RECORDS).getValue();
+ getContext().getStatisticsReporter().reportItemsProcessed(outputRecords);
+
return returnEvents;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
index 30f78fe..19eb18a 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
@@ -72,7 +72,7 @@ import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
-
+@SuppressWarnings({ "rawtypes", "unchecked" })
@RunWith(Parameterized.class)
public class TestOnFileSortedOutput {
private static final Random rnd = new Random();
@@ -90,6 +90,7 @@ public class TestOnFileSortedOutput {
private int sorterThreads;
final AtomicLong outputSize = new AtomicLong();
+ final AtomicLong numRecords = new AtomicLong();
private KeyValuesWriter writer;
private OrderedPartitionedKVOutput sortedOutput;
@@ -135,6 +136,7 @@ public class TestOnFileSortedOutput {
sendEmptyPartitionViaEvent);
outputSize.set(0);
+ numRecords.set(0);
fs.mkdirs(workingDir);
this.partitions = Math.max(1, rnd.nextInt(10));
}
@@ -271,10 +273,12 @@ public class TestOnFileSortedOutput {
startSortedOutput(partitions);
//Write random set of keys
+ long recordsWritten = numRecords.get();
for (int i = 0; i < Math.max(1, rnd.nextInt(50)); i++) {
Text key = new Text(new BigInteger(256, rnd).toString());
LinkedList values = new LinkedList();
for (int j = 0; j < Math.max(2, rnd.nextInt(10)); j++) {
+ recordsWritten++;
values.add(new Text(new BigInteger(256, rnd).toString()));
}
writer.write(key, values);
@@ -282,7 +286,7 @@ public class TestOnFileSortedOutput {
List<Event> eventList = sortedOutput.close();
assertTrue(eventList != null && eventList.size() == 2);
-
+ assertEquals(recordsWritten, numRecords.get());
ShuffleUserPayloads.DataMovementEventPayloadProto
payload = ShuffleUserPayloads.DataMovementEventPayloadProto
.parseFrom(
@@ -358,6 +362,13 @@ public class TestOnFileSortedOutput {
return null;
}
}).when(reporter).reportDataSize(anyLong());
+ doAnswer(new Answer() {
+ @Override public Object answer(InvocationOnMock invocation) throws Throwable {
+ numRecords.set((Long) invocation.getArguments()[0]);
+ return null;
+ }
+ }).when(reporter).reportItemsProcessed(anyLong());
+
OutputContext context = mock(OutputContext.class);
doReturn(counters).when(context).getCounters();
http://git-wip-us.apache.org/repos/asf/tez/blob/f46997a7/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java
index 41eb9a4..2b25daf 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java
@@ -81,6 +81,7 @@ import org.mockito.ArgumentCaptor;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+@SuppressWarnings("rawtypes")
public class TestOnFileUnorderedKVOutput {
private static final Logger LOG = LoggerFactory.getLogger(TestOnFileUnorderedKVOutput.class);
@@ -139,6 +140,7 @@ public class TestOnFileUnorderedKVOutput {
events = kvOutput.close();
assertEquals(45, stats.getIOStatistics().values().iterator().next().getDataSize());
+ assertEquals(5, stats.getIOStatistics().values().iterator().next().getItemsProcessed());
assertTrue(events != null && events.size() == 1);
CompositeDataMovementEvent dmEvent = (CompositeDataMovementEvent)events.get(0);
[23/50] [abbrv] tez git commit: TEZ-2292. Add e2e test for error
reporting when vertex manager invokes plugin APIs (bikas)
Posted by ss...@apache.org.
TEZ-2292. Add e2e test for error reporting when vertex manager invokes plugin APIs (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c6e400e2
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c6e400e2
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c6e400e2
Branch: refs/heads/TEZ-2003
Commit: c6e400e2d1d484bc5bc33ec27324b065de40e465
Parents: f46997a
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Apr 21 16:48:54 2015 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Apr 21 16:48:54 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../tez/dag/api/VertexManagerPluginContext.java | 3 +-
.../dag/app/dag/impl/AMUserCodeException.java | 3 +-
.../tez/dag/app/dag/impl/VertexManager.java | 7 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 98 +++++++++-----------
.../tez/test/VertexManagerPluginForTest.java | 70 +++++++++++++-
.../vertexmanager/InputReadyVertexManager.java | 8 +-
.../vertexmanager/ShuffleVertexManager.java | 9 +-
.../vertexmanager/TestShuffleVertexManager.java | 23 ++---
9 files changed, 129 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/c6e400e2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5293120..0c83c08 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,8 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2292. Add e2e test for error reporting when vertex manager invokes
+ plugin APIs
TEZ-2308. Add set/get of record counts in task/vertex statistics
TEZ-2344. Tez UI: Equip basic-ember-table's cell level loading for all use cases in all DAGs table
TEZ-2313. Regression in handling obsolete events in ShuffleScheduler.
http://git-wip-us.apache.org/repos/asf/tez/blob/c6e400e2/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
index afcdb88..8b0e89e 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
@@ -172,11 +172,10 @@ public interface VertexManagerPluginContext {
* Map with Key=name of {@link Edge} to be updated and Value=
* {@link EdgeProperty}. The name of the Edge will be the
* corresponding source vertex name.
- * @throws TezException Exception to indicate errors
*/
public void reconfigureVertex(int parallelism,
@Nullable VertexLocationHint locationHint,
- @Nullable Map<String, EdgeProperty> sourceEdgeProperties) throws TezException;
+ @Nullable Map<String, EdgeProperty> sourceEdgeProperties);
/**
* Allows a VertexManagerPlugin to assign Events for Root Inputs
http://git-wip-us.apache.org/repos/asf/tez/blob/c6e400e2/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java
index b4e1849..22b1211 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java
@@ -19,6 +19,7 @@
package org.apache.tez.dag.app.dag.impl;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.tez.dag.api.TezException;
/**
@@ -28,7 +29,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
* <li>InputInitializer</li>
*/
@Private
-public class AMUserCodeException extends Exception {
+public class AMUserCodeException extends TezException {
private static final long serialVersionUID = -3642816091492797520L;
http://git-wip-us.apache.org/repos/asf/tez/blob/c6e400e2/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
index bcea22c..2ac1acf 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
@@ -44,7 +44,6 @@ import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.InputInitializerDescriptor;
import org.apache.tez.dag.api.RootInputLeafOutput;
-import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.VertexLocationHint;
@@ -171,7 +170,6 @@ public class VertexManager {
managedVertex.setParallelism(parallelism, vertexLocationHint, sourceEdgeManagers,
rootInputSpecUpdate, true);
} catch (AMUserCodeException e) {
- // workaround: convert it to TezUncheckedException which would be caught in VM
throw new TezUncheckedException(e);
}
}
@@ -179,13 +177,12 @@ public class VertexManager {
@Override
public synchronized void reconfigureVertex(int parallelism,
@Nullable VertexLocationHint locationHint,
- @Nullable Map<String, EdgeProperty> sourceEdgeProperties) throws TezException {
+ @Nullable Map<String, EdgeProperty> sourceEdgeProperties) {
checkAndThrowIfDone();
try {
managedVertex.reconfigureVertex(parallelism, locationHint, sourceEdgeProperties);
} catch (AMUserCodeException e) {
- // convert it to TezException which would be caught in VM
- throw new TezException(e);
+ throw new TezUncheckedException(e);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/c6e400e2/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index c752965..2403599 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -170,6 +170,7 @@ import org.apache.tez.runtime.api.events.TaskAttemptFailedEvent;
import org.apache.tez.runtime.api.events.VertexManagerEvent;
import org.apache.tez.test.EdgeManagerForTest;
import org.apache.tez.test.VertexManagerPluginForTest;
+import org.apache.tez.test.VertexManagerPluginForTest.VertexManagerPluginForTestConfig;
import org.apache.tez.runtime.api.impl.EventMetaData;
import org.apache.tez.runtime.api.impl.EventMetaData.EventProducerConsumerType;
import org.apache.tez.runtime.api.impl.GroupInputSpec;
@@ -184,6 +185,7 @@ import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.internal.util.collections.Sets;
+import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ListenableFuture;
@@ -192,6 +194,7 @@ import com.google.common.util.concurrent.ListeningExecutorService;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+@SuppressWarnings("unchecked")
public class TestVertexImpl {
private static final Logger LOG = LoggerFactory.getLogger(TestVertexImpl.class);
@@ -2273,7 +2276,6 @@ public class TestVertexImpl {
}
}
- @SuppressWarnings("unchecked")
private void initVertex(VertexImpl v) {
Assert.assertEquals(VertexState.NEW, v.getState());
dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(),
@@ -2285,7 +2287,6 @@ public class TestVertexImpl {
startVertex(v, true);
}
- @SuppressWarnings("unchecked")
private void killVertex(VertexImpl v) {
dispatcher.getEventHandler().handle(
new VertexEventTermination(v.getVertexId(), VertexTerminationCause.DAG_KILL));
@@ -2294,7 +2295,6 @@ public class TestVertexImpl {
Assert.assertEquals(v.getTerminationCause(), VertexTerminationCause.DAG_KILL);
}
- @SuppressWarnings("unchecked")
private void startVertex(VertexImpl v,
boolean checkRunningState) {
Assert.assertEquals(VertexState.INITED, v.getState());
@@ -2379,7 +2379,6 @@ public class TestVertexImpl {
updateTracker.unregisterForVertexUpdates("vertex3", listener);
}
- @SuppressWarnings("unchecked")
@Test (timeout=5000)
public void testVertexConfigureEventWithReconfigure() throws Exception {
useCustomInitializer = true;
@@ -2626,7 +2625,6 @@ public class TestVertexImpl {
}
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexPendingTaskEvents() {
initAllVertices(VertexState.INITED);
@@ -2701,7 +2699,6 @@ public class TestVertexImpl {
((EdgeManagerForTest) modifiedEdgeManager).getUserPayload().deepCopyAsArray()));
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testBasicVertexCompletion() {
initAllVertices(VertexState.INITED);
@@ -2725,7 +2722,6 @@ public class TestVertexImpl {
Assert.assertEquals(2, v.getCompletedTasks());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
@Ignore // FIXME fix verteximpl for this test to work
public void testDuplicateTaskCompletion() {
@@ -2753,8 +2749,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
}
-
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexFailure() {
initAllVertices(VertexState.INITED);
@@ -2801,7 +2795,6 @@ public class TestVertexImpl {
"vertex received kill while in running state"));
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexKillPending() {
initAllVertices(VertexState.INITED);
@@ -2827,7 +2820,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.KILLED, v.getState());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexKill() {
initAllVertices(VertexState.INITED);
@@ -2853,7 +2845,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.KILLED, v.getState());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testKilledTasksHandling() {
initAllVertices(VertexState.INITED);
@@ -2895,7 +2886,6 @@ public class TestVertexImpl {
instanceof ShuffleVertexManager);
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexTaskFailure() {
initAllVertices(VertexState.INITED);
@@ -2926,7 +2916,6 @@ public class TestVertexImpl {
Assert.assertEquals(1, committer.abortCounter);
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexTaskAttemptProcessorFailure() throws Exception {
initAllVertices(VertexState.INITED);
@@ -2962,7 +2951,6 @@ public class TestVertexImpl {
Assert.assertEquals(TaskAttemptTerminationCause.APPLICATION_ERROR, ta.getTerminationCause());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexTaskAttemptInputFailure() throws Exception {
initAllVertices(VertexState.INITED);
@@ -2999,7 +2987,6 @@ public class TestVertexImpl {
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexTaskAttemptOutputFailure() throws Exception {
initAllVertices(VertexState.INITED);
@@ -3098,7 +3085,6 @@ public class TestVertexImpl {
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGEventGeneration() {
initAllVertices(VertexState.INITED);
@@ -3120,7 +3106,6 @@ public class TestVertexImpl {
DAGEventType.DAG_VERTEX_COMPLETED).intValue());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testTaskReschedule() {
// For downstream failures
@@ -3150,8 +3135,7 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
}
-
- @SuppressWarnings("unchecked")
+
@Test(timeout = 5000)
public void testVertexSuccessToRunningAfterTaskScheduler() {
// For downstream failures
@@ -3191,8 +3175,7 @@ public class TestVertexImpl {
dagEventDispatcher.eventCount.get(
DAGEventType.DAG_VERTEX_COMPLETED).intValue());
}
-
- @SuppressWarnings("unchecked")
+
@Test(timeout = 5000)
public void testVertexSuccessToFailedAfterTaskScheduler() throws Exception {
// For downstream failures
@@ -3240,7 +3223,6 @@ public class TestVertexImpl {
DAGEventType.DAG_VERTEX_COMPLETED).intValue());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexCommit() {
initAllVertices(VertexState.INITED);
@@ -3267,7 +3249,6 @@ public class TestVertexImpl {
Assert.assertEquals(1, committer.setupCounter);
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testTaskFailedAfterVertexSuccess() {
initAllVertices(VertexState.INITED);
@@ -3303,7 +3284,6 @@ public class TestVertexImpl {
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testBadCommitter() throws Exception {
VertexImpl v = vertices.get("vertex2");
@@ -3348,7 +3328,6 @@ public class TestVertexImpl {
Assert.assertEquals(1, committer.setupCounter);
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testBadCommitter2() throws Exception {
VertexImpl v = vertices.get("vertex2");
@@ -3391,8 +3370,7 @@ public class TestVertexImpl {
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
-
- @SuppressWarnings("unchecked")
+
@Test(timeout = 5000)
public void testVertexInitWithCustomVertexManager() throws Exception {
setupPreDagCreation();
@@ -3612,7 +3590,39 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex4").getState());
}
- @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
+ public void testVertexVMErrorReport() throws Exception {
+ int numTasks = 5;
+ // create a diamond shaped dag with 1-1 edges.
+ setupPreDagCreation();
+ dagPlan = createDAGPlanForOneToOneSplit(null, numTasks, false);
+ setupPostDagCreation();
+ VertexImpl v1 = vertices.get("vertex1");
+ initAllVertices(VertexState.INITED);
+
+ // fudge vertex manager so that tasks dont start running
+ // it is not calling reconfigurtionPlanned() but will call reconfigureVertex().
+ // the vertex is already fully configured. this causes exception and verify that
+ // its caught and reported.
+ VertexManagerPluginForTestConfig config = new VertexManagerPluginForTestConfig();
+ config.setReconfigureOnStart(true);
+ v1.vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(
+ VertexManagerPluginForTest.class.getName()).setUserPayload(
+ UserPayload.create(config.getPayload())), UserGroupInformation.getCurrentUser(), v1,
+ appContext, mock(StateChangeNotifier.class));
+ v1.vertexManager.initialize();
+
+ startVertex(v1, false);
+ dispatcher.await();
+
+ // failed due to exception
+ Assert.assertEquals(VertexState.FAILED, vertices.get("vertex1").getState());
+ Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, vertices.get("vertex1")
+ .getTerminationCause());
+ Assert.assertTrue(Joiner.on(":").join(vertices.get("vertex1").getDiagnostics()).contains(
+ "context.vertexReconfigurationPlanned() before re-configuring"));
+ }
+
@Test(timeout = 5000)
public void testInvalidEvent() {
VertexImpl v = vertices.get("vertex2");
@@ -3625,7 +3635,6 @@ public class TestVertexImpl {
DAGEventType.INTERNAL_ERROR).intValue());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexWithInitializerFailure() throws Exception {
useCustomInitializer = true;
@@ -4125,7 +4134,6 @@ public class TestVertexImpl {
initializer.stateUpdates.get(2).getVertexState());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testInputInitializerEvents() throws Exception {
useCustomInitializer = true;
@@ -4202,7 +4210,6 @@ public class TestVertexImpl {
Assert.assertEquals(0, initializerWrapper.getPendingEvents().get(v1.getName()).size());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
/**
* Ref: TEZ-1494
@@ -4563,7 +4570,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.INITED, v1.getState());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 500000)
public void testVertexWithInitializerSuccess() throws Exception {
useCustomInitializer = true;
@@ -4637,8 +4643,7 @@ public class TestVertexImpl {
Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
}
}
-
- @SuppressWarnings("unchecked")
+
@Test(timeout = 5000)
public void testVertexWithInputDistributor() throws Exception {
useCustomInitializer = true;
@@ -4674,7 +4679,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.INITED, v2.getState());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexRootInputSpecUpdateAll() throws Exception {
useCustomInitializer = true;
@@ -4704,8 +4708,7 @@ public class TestVertexImpl {
Assert.assertEquals(4, inputSpecs.get(0).getPhysicalEdgeCount());
}
}
-
- @SuppressWarnings("unchecked")
+
@Test(timeout = 5000)
public void testVertexRootInputSpecUpdatePerTask() throws Exception {
useCustomInitializer = true;
@@ -4983,7 +4986,6 @@ public class TestVertexImpl {
}
}
- @SuppressWarnings("unchecked")
@Test(timeout=5000)
public void testVertexGroupInput() throws AMUserCodeException {
setupPreDagCreation();
@@ -5010,8 +5012,7 @@ public class TestVertexImpl {
assertTrue(groupInSpec.get(0).getGroupVertices().contains("B"));
groupInSpec.get(0).getMergedInputDescriptor().getClassName().equals("Group.class");
}
-
- @SuppressWarnings("unchecked")
+
@Test(timeout = 5000)
public void testStartWithUninitializedCustomEdge() throws Exception {
// Race when a source vertex manages to start before the target vertex has
@@ -5060,8 +5061,7 @@ public class TestVertexImpl {
Assert.assertNotNull(vB.getTask(0));
Assert.assertNotNull(vC.getTask(0));
}
-
- @SuppressWarnings("unchecked")
+
@Test(timeout = 5000)
public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception {
// Race when a source vertex manages to start before the target vertex has
@@ -5136,7 +5136,6 @@ public class TestVertexImpl {
Assert.assertNotNull(vC.getTask(0));
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testInitStartRace() throws AMUserCodeException {
// Race when a source vertex manages to start before the target vertex has
@@ -5160,7 +5159,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.RUNNING, vC.getState());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testInitStartRace2() throws AMUserCodeException {
// Race when a source vertex manages to start before the target vertex has
@@ -5188,7 +5186,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.RUNNING, vC.getState());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromVM_Initialize() throws AMUserCodeException {
useCustomInitializer = true;
@@ -5208,7 +5205,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromVM_OnRootVertexInitialized() throws Exception {
useCustomInitializer = true;
@@ -5233,7 +5229,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromVM_OnVertexStarted() throws Exception {
useCustomInitializer = true;
@@ -5261,7 +5256,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromVM_OnSourceTaskCompleted() throws Exception {
useCustomInitializer = true;
@@ -5298,7 +5292,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v2.getTerminationCause());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromVM_OnVertexManagerEventReceived() throws Exception {
useCustomInitializer = true;
@@ -5325,7 +5318,6 @@ public class TestVertexImpl {
Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexManagerEventReceived.name()));
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromVM_OnVertexManagerVertexStateUpdated() throws Exception {
useCustomInitializer = true;
@@ -5376,7 +5368,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v1.getTerminationCause());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromII_InitFailedAfterInitialized() throws Exception {
useCustomInitializer = true;
@@ -5401,7 +5392,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v1.getTerminationCause());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromII_InitFailedAfterRunning() throws Exception {
useCustomInitializer = true;
@@ -5427,7 +5417,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v1.getTerminationCause());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromII_HandleInputInitializerEvent() throws Exception {
useCustomInitializer = true;
@@ -5507,7 +5496,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v2.getTerminationCause());
}
- @SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromII_InitSucceededAfterInitFailure() throws AMUserCodeException, InterruptedException {
useCustomInitializer = true;
http://git-wip-us.apache.org/repos/asf/tez/blob/c6e400e2/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java b/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java
index 422d785..84e060b 100644
--- a/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java
+++ b/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java
@@ -18,26 +18,92 @@
package org.apache.tez.test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.VertexManagerPlugin;
import org.apache.tez.dag.api.VertexManagerPluginContext;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.events.VertexManagerEvent;
public class VertexManagerPluginForTest extends VertexManagerPlugin {
+ VertexManagerPluginForTestConfig pluginConfig = new VertexManagerPluginForTestConfig();
+
+ public static class VertexManagerPluginForTestConfig {
+ Configuration conf = new Configuration(false);
+ static final String RECONFIGURE_ON_START = "reconfigureOnStart";
+ static final String NUM_TASKS = "numTasks";
+
+ public void setReconfigureOnStart(boolean value) {
+ conf.setBoolean(RECONFIGURE_ON_START, value);
+ }
+
+ public void setNumTasks(int value) {
+ conf.setInt(NUM_TASKS, value);
+ }
+
+ boolean getReconfigureOnStart() {
+ return conf.getBoolean(RECONFIGURE_ON_START, false);
+ }
+
+ int getNumTasks() {
+ return conf.getInt(NUM_TASKS, 1);
+ }
+
+ public ByteBuffer getPayload() {
+ ByteArrayOutputStream b = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(b);
+ try {
+ conf.write(out);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ return ByteBuffer.wrap(b.toByteArray());
+ }
+
+ void initialize(ByteBuffer buff) {
+ ByteBuffer copy = ByteBuffer.allocate(buff.capacity());
+ copy.put(buff);
+ copy.flip();
+ ByteArrayInputStream b = new ByteArrayInputStream(copy.array());
+ DataInputStream in = new DataInputStream(b);
+ try {
+ conf.readFields(in);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+ }
public VertexManagerPluginForTest(VertexManagerPluginContext context) {
super(context);
}
+
@Override
- public void initialize() {}
+ public void initialize() {
+ UserPayload payload = getContext().getUserPayload();
+ if (payload != null && payload.getPayload() != null) {
+ pluginConfig.initialize(getContext().getUserPayload().getPayload());
+ }
+ }
@Override
- public void onVertexStarted(Map<String, List<Integer>> completions) {}
+ public void onVertexStarted(Map<String, List<Integer>> completions) {
+ if (pluginConfig.getReconfigureOnStart()) {
+ getContext().reconfigureVertex(pluginConfig.getNumTasks(), null, null);
+ }
+ }
@Override
public void onSourceTaskCompleted(String srcVertexName, Integer taskId) {}
http://git-wip-us.apache.org/repos/asf/tez/blob/c6e400e2/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java
index 8671161..30e3e81 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
import org.apache.tez.dag.api.InputDescriptor;
-import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.TaskLocationHint;
import org.apache.tez.dag.api.VertexManagerPlugin;
@@ -124,12 +123,7 @@ public class InputReadyVertexManager extends VertexManagerPlugin {
// must change parallelism to make them the same
LOG.info("Update parallelism of vertex: " + getContext().getVertexName() +
" to " + oneToOneSrcTaskCount + " to match source 1-1 vertices.");
- try {
- getContext().reconfigureVertex(oneToOneSrcTaskCount, null, null);
- } catch (TezException e) {
- // TODO fail vertex - TEZ-2292
- LOG.warn("Failed to change parallelism in: " + getContext().getVertexName(), e);
- }
+ getContext().reconfigureVertex(oneToOneSrcTaskCount, null, null);
}
oneToOneSrcTasksDoneCount = new int[oneToOneSrcTaskCount];
oneToOneLocationHints = new TaskLocationHint[oneToOneSrcTaskCount];
http://git-wip-us.apache.org/repos/asf/tez/blob/c6e400e2/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
index 9be9986..2ea0299 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
@@ -39,7 +39,6 @@ import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
import org.apache.tez.dag.api.InputDescriptor;
-import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.VertexManagerPlugin;
@@ -533,12 +532,8 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
edgeProperties.put(vertex, newEdgeProp);
}
- try {
- getContext().reconfigureVertex(finalTaskParallelism, null, edgeProperties);
- } catch (TezException e) {
- // TODO fail vertex - TEZ-2292
- LOG.warn("Failed to change parallelism in: " + getContext().getVertexName(), e);
- }
+ getContext().reconfigureVertex(finalTaskParallelism, null, edgeProperties);
+
updatePendingTasks();
}
return true;
http://git-wip-us.apache.org/repos/asf/tez/blob/c6e400e2/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
index 27cd292..8807674 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
@@ -30,7 +30,6 @@ import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
-import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.VertexLocationHint;
@@ -66,9 +65,9 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+@SuppressWarnings({ "unchecked", "rawtypes" })
public class TestShuffleVertexManager {
- @SuppressWarnings({ "unchecked", "rawtypes" })
@Test(timeout = 5000)
public void testShuffleVertexManagerAutoParallelism() throws Exception {
Configuration conf = new Configuration();
@@ -452,7 +451,6 @@ public class TestShuffleVertexManager {
}
}
- @SuppressWarnings({ "unchecked", "rawtypes" })
@Test(timeout = 5000)
public void testShuffleVertexManagerSlowStart() {
Configuration conf = new Configuration();
@@ -779,18 +777,13 @@ public class TestShuffleVertexManager {
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 9);
//Ensure that setVertexParallelism is not called for R2.
- try {
- verify(mockContext_R2, times(0)).reconfigureVertex(anyInt(), any(VertexLocationHint.class),
- anyMap());
- // complete configuration of r1 triggers the scheduling
- manager.onVertexStateUpdated(new VertexStateUpdate(r1, VertexState.CONFIGURED));
- verify(mockContext_R2, times(1)).reconfigureVertex(eq(1), any(VertexLocationHint.class),
- anyMap());
- } catch (TezException e) {
- e.printStackTrace();
- Assert.fail(); // should not happen
- }
-
+ verify(mockContext_R2, times(0)).reconfigureVertex(anyInt(), any(VertexLocationHint.class),
+ anyMap());
+ // complete configuration of r1 triggers the scheduling
+ manager.onVertexStateUpdated(new VertexStateUpdate(r1, VertexState.CONFIGURED));
+ verify(mockContext_R2, times(1)).reconfigureVertex(eq(1), any(VertexLocationHint.class),
+ anyMap());
+
Assert.assertTrue(manager.pendingTasks.size() == 0); // all tasks scheduled
Assert.assertTrue(scheduledTasks.size() == 3);
[07/50] [abbrv] tez git commit: TEZ-2323. Fix TestOrderedWordcount to
use MR memory configs. (hitesh)
Posted by ss...@apache.org.
TEZ-2323. Fix TestOrderedWordcount to use MR memory configs. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/5de0fb12
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/5de0fb12
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/5de0fb12
Branch: refs/heads/TEZ-2003
Commit: 5de0fb123991b716a810ce7d122faf474fe64243
Parents: 19378d5
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Apr 15 17:59:15 2015 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed Apr 15 17:59:15 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/mapreduce/examples/TestOrderedWordCount.java | 6 +++---
.../src/test/java/org/apache/tez/test/TestSecureShuffle.java | 3 ++-
3 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/5de0fb12/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 56f6ea9..4cd4557 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2323. Fix TestOrderedWordcount to use MR memory configs.
TEZ-1482. Fix memory issues for Local Mode running concurrent tasks
TEZ-2033. Update TestOrderedWordCount to add processor configs as history text
and use MR configs correctly
http://git-wip-us.apache.org/repos/asf/tez/blob/5de0fb12/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
index 4a05dab..eb56795 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
@@ -283,7 +283,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
OrderedPartitionedKVEdgeConfig edgeConf1 = OrderedPartitionedKVEdgeConfig
.newBuilder(Text.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName()).setFromConfiguration(conf)
+ HashPartitioner.class.getName()).setFromConfiguration(iReduceStageConf)
.configureInput().useLegacyInput().done().build();
dag.addEdge(
Edge.create(dag.getVertex("initialmap"), dag.getVertex("intermediate_reducer"),
@@ -291,7 +291,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
OrderedPartitionedKVEdgeConfig edgeConf2 = OrderedPartitionedKVEdgeConfig
.newBuilder(IntWritable.class.getName(), Text.class.getName(),
- HashPartitioner.class.getName()).setFromConfiguration(conf)
+ HashPartitioner.class.getName()).setFromConfiguration(finalReduceConf)
.configureInput().useLegacyInput().done().build();
dag.addEdge(
Edge.create(dag.getVertex("intermediate_reducer"), dag.getVertex("finalreduce"),
@@ -443,7 +443,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
Map<String, LocalResource> localResources =
new TreeMap<String, LocalResource>();
- DAG dag = instance.createDAG(fs, conf, localResources,
+ DAG dag = instance.createDAG(fs, tezConf, localResources,
stagingDir, dagIndex, inputPath, outputPath,
generateSplitsInClient, useMRSettings, intermediateNumReduceTasks);
http://git-wip-us.apache.org/repos/asf/tez/blob/5de0fb12/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
index 37a8ef6..a0f7d8b 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
@@ -140,7 +140,8 @@ public class TestSecureShuffle {
TestOrderedWordCount wordCount = new TestOrderedWordCount();
wordCount.setConf(new Configuration(miniTezCluster.getConfig()));
- String[] args = new String[] { inputLoc.toString(), outputLoc.toString() };
+ String[] args = new String[] { "-DUSE_MR_CONFIGS=false",
+ inputLoc.toString(), outputLoc.toString() };
assertEquals(expectedResult, wordCount.run(args));
}
[42/50] [abbrv] tez git commit: TEZ-2187. Allow TaskCommunicators to
report failed / killed attempts. (sseth)
Posted by ss...@apache.org.
TEZ-2187. Allow TaskCommunicators to report failed / killed attempts. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/e901aeb6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/e901aeb6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/e901aeb6
Branch: refs/heads/TEZ-2003
Commit: e901aeb6664d7672f063db0edd6f38c99d86fd22
Parents: 6fa456a
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Mar 10 01:25:39 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:47:33 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../org/apache/tez/common/TezUtilsInternal.java | 60 +++++++++
.../tez/dag/api/TaskAttemptEndReason.java | 24 ++++
.../records/TaskAttemptTerminationCause.java | 7 +-
.../apache/tez/dag/api/TaskCommunicator.java | 2 +
.../tez/dag/api/TaskCommunicatorContext.java | 13 +-
.../dag/app/TaskAttemptListenerImpTezDag.java | 33 +++++
.../event/TaskAttemptEventAttemptFailed.java | 2 +
.../event/TaskAttemptEventAttemptKilled.java | 47 +++++++
.../dag/app/dag/event/TaskAttemptEventType.java | 5 +-
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 33 ++++-
.../tez/dag/app/rm/AMSchedulerEventTAEnded.java | 9 +-
.../dag/app/rm/LocalTaskSchedulerService.java | 3 +-
.../dag/app/rm/TaskSchedulerEventHandler.java | 7 +-
.../tez/dag/app/rm/TaskSchedulerService.java | 6 +-
.../dag/app/rm/YarnTaskSchedulerService.java | 8 +-
.../app/TestTaskAttemptListenerImplTezDag.java | 1 +
.../app/TestTaskAttemptListenerImplTezDag2.java | 126 +++++++++++++++++++
.../tez/dag/app/rm/TestContainerReuse.java | 65 +++++-----
.../app/rm/TestLocalTaskSchedulerService.java | 5 +-
.../tez/dag/app/rm/TestTaskScheduler.java | 18 +--
.../rm/TezTestServiceTaskSchedulerService.java | 3 +-
.../TezTestServiceTaskCommunicatorImpl.java | 36 +++++-
.../org/apache/tez/service/ContainerRunner.java | 5 +-
.../tez/service/MiniTezTestServiceCluster.java | 5 +-
.../tez/service/impl/ContainerRunnerImpl.java | 60 +++++++--
.../apache/tez/service/impl/TezTestService.java | 6 +-
.../impl/TezTestServiceProtocolServerImpl.java | 10 +-
.../tez/tests/TestExternalTezServices.java | 29 +++++
29 files changed, 548 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 7726815..774a685 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -10,5 +10,6 @@ ALL CHANGES:
TEZ-2138. Fix minor bugs in adding default scheduler, getting launchers.
TEZ-2139. Update tez version to 0.7.0-TEZ-2003-SNAPSHOT.
TEZ-2175. Task priority should be available to the TaskCommunicator plugin.
+ TEZ-2187. Allow TaskCommunicators to report failed / killed attempts.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
index 9c78377..347a4f6 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.log4j.Appender;
import org.apache.tez.dag.api.DagTypeConverters;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.records.DAGProtos;
import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
@@ -49,6 +50,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Stopwatch;
+import org.apache.tez.dag.records.TaskAttemptTerminationCause;
@Private
public class TezUtilsInternal {
@@ -234,4 +236,62 @@ public class TezUtilsInternal {
return sb.toString();
}
+ public static TaskAttemptTerminationCause fromTaskAttemptEndReason(
+ TaskAttemptEndReason taskAttemptEndReason) {
+ if (taskAttemptEndReason == null) {
+ return null;
+ }
+ switch (taskAttemptEndReason) {
+ case COMMUNICATION_ERROR:
+ return TaskAttemptTerminationCause.COMMUNICATION_ERROR;
+ case SERVICE_BUSY:
+ return TaskAttemptTerminationCause.SERVICE_BUSY;
+ case INTERRUPTED_BY_SYSTEM:
+ return TaskAttemptTerminationCause.INTERRUPTED_BY_SYSTEM;
+ case INTERRUPTED_BY_USER:
+ return TaskAttemptTerminationCause.INTERRUPTED_BY_USER;
+ case OTHER:
+ return TaskAttemptTerminationCause.UNKNOWN_ERROR;
+ default:
+ return TaskAttemptTerminationCause.UNKNOWN_ERROR;
+ }
+ }
+
+ public static TaskAttemptEndReason toTaskAttemptEndReason(TaskAttemptTerminationCause cause) {
+ // TODO Post TEZ-2003. Consolidate these states, and mappings.
+ if (cause == null) {
+ return null;
+ }
+ switch (cause) {
+ case COMMUNICATION_ERROR:
+ return TaskAttemptEndReason.COMMUNICATION_ERROR;
+ case SERVICE_BUSY:
+ return TaskAttemptEndReason.SERVICE_BUSY;
+ case INTERRUPTED_BY_SYSTEM:
+ return TaskAttemptEndReason.INTERRUPTED_BY_SYSTEM;
+ case INTERRUPTED_BY_USER:
+ return TaskAttemptEndReason.INTERRUPTED_BY_USER;
+ case UNKNOWN_ERROR:
+ case TERMINATED_BY_CLIENT:
+ case TERMINATED_AT_SHUTDOWN:
+ case INTERNAL_PREEMPTION:
+ case EXTERNAL_PREEMPTION:
+ case TERMINATED_INEFFECTIVE_SPECULATION:
+ case TERMINATED_EFFECTIVE_SPECULATION:
+ case TERMINATED_ORPHANED:
+ case APPLICATION_ERROR:
+ case FRAMEWORK_ERROR:
+ case INPUT_READ_ERROR:
+ case OUTPUT_WRITE_ERROR:
+ case OUTPUT_LOST:
+ case TASK_HEARTBEAT_ERROR:
+ case CONTAINER_LAUNCH_FAILED:
+ case CONTAINER_EXITED:
+ case CONTAINER_STOPPED:
+ case NODE_FAILED:
+ case NODE_DISK_ERROR:
+ default:
+ return TaskAttemptEndReason.OTHER;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-common/src/main/java/org/apache/tez/dag/api/TaskAttemptEndReason.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/api/TaskAttemptEndReason.java b/tez-common/src/main/java/org/apache/tez/dag/api/TaskAttemptEndReason.java
new file mode 100644
index 0000000..96a4768
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/api/TaskAttemptEndReason.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+// TODO TEZ-2003 Expose as a public API
+public enum TaskAttemptEndReason {
+ COMMUNICATION_ERROR,
+ SERVICE_BUSY,
+ INTERRUPTED_BY_SYSTEM,
+ INTERRUPTED_BY_USER,
+ OTHER
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptTerminationCause.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptTerminationCause.java b/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptTerminationCause.java
index ef0bb33..7112d9e 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptTerminationCause.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptTerminationCause.java
@@ -41,5 +41,10 @@ public enum TaskAttemptTerminationCause {
CONTAINER_STOPPED, // Container stopped or released by Tez
NODE_FAILED, // Node for the container failed
NODE_DISK_ERROR, // Disk failed on the node runnign the task
-
+
+ COMMUNICATION_ERROR, // Equivalent to a launch failure
+ SERVICE_BUSY, // Service rejected the task
+ INTERRUPTED_BY_SYSTEM, // Interrupted by the system. e.g. Pre-emption
+ INTERRUPTED_BY_USER, // Interrupted by the user
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
index 82eed20..945091e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
@@ -45,6 +45,8 @@ public abstract class TaskCommunicator extends AbstractService {
Credentials credentials,
boolean credentialsChanged, int priority);
+ // TODO TEZ-2003. Are additional APIs required to mark a container as completed ? - for completeness.
+
// TODO TEZ-2003 Remove reference to TaskAttemptID
public abstract void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID);
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
index 41675fe..a85fb7f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
@@ -14,6 +14,7 @@
package org.apache.tez.dag.api;
+import javax.annotation.Nullable;
import java.io.IOException;
import org.apache.hadoop.security.Credentials;
@@ -37,15 +38,21 @@ public interface TaskCommunicatorContext {
// TODO TEZ-2003 Move to vertex, taskIndex, version
boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException;
+ // TODO TEZ-2003 Split the heartbeat API to a liveness check and a status update
TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request) throws IOException, TezException;
boolean isKnownContainer(ContainerId containerId);
- // TODO TEZ-2003 Move to vertex, taskIndex, version
+ // TODO TEZ-2003 Move to vertex, taskIndex, version. Rename to taskAttempt*
void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId);
- // TODO TEZ-2003 Add an API to register task failure - for example, a communication failure.
- // This will have to take into consideration the TA_FAILED event
+ // TODO TEZ-2003 Move to vertex, taskIndex, version. Rename to taskAttempt*
+ void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason, @Nullable String diagnostics);
+
+ // TODO TEZ-2003 Move to vertex, taskIndex, version. Rename to taskAttempt*
+ void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason, @Nullable String diagnostics);
+
+ // TODO TEZ-2003 API. Should a method exist for task succeeded.
// TODO Eventually Add methods to report availability stats to the scheduler.
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index ee5d4ee..9f33a1f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -26,14 +26,17 @@ import java.util.concurrent.ConcurrentMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.common.TezUtilsInternal;
import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
import org.apache.tez.dag.api.TaskHeartbeatResponse;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezException;
@@ -43,7 +46,10 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.dag.api.TaskHeartbeatRequest;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Task;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
import org.apache.tez.dag.app.rm.container.AMContainerTask;
import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -234,6 +240,33 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
pingContainerHeartbeatHandler(containerId);
}
+ @Override
+ public void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
+ String diagnostics) {
+ // Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
+ // and messages from the scheduler will release the container.
+ // TODO TEZ-2003 Maybe consider un-registering here itself, since the task is not active anymore,
+ // instead of waiting for the unregister to flow through the Container.
+ // Fix along the same lines as TEZ-2124 by introducing an explict context.
+ context.getEventHandler().handle(new TaskAttemptEventAttemptKilled(taskAttemptId,
+ diagnostics, TezUtilsInternal.fromTaskAttemptEndReason(
+ taskAttemptEndReason)));
+ }
+
+ @Override
+ public void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
+ String diagnostics) {
+ // Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
+ // and messages from the scheduler will release the container.
+ // TODO TEZ-2003 Maybe consider un-registering here itself, since the task is not active anymore,
+ // instead of waiting for the unregister to flow through the Container.
+ // Fix along the same lines as TEZ-2124 by introducing an explict context.
+ context.getEventHandler().handle(new TaskAttemptEventAttemptFailed(taskAttemptId,
+ TaskAttemptEventType.TA_FAILED, diagnostics, TezUtilsInternal.fromTaskAttemptEndReason(
+ taskAttemptEndReason)));
+ }
+
+
/**
* Child checking whether it can commit.
* <p/>
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java
index b9c1d09..7ec8921 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java
@@ -26,6 +26,8 @@ public class TaskAttemptEventAttemptFailed extends TaskAttemptEvent
private final String diagnostics;
private final TaskAttemptTerminationCause errorCause;
+
+ /* Accepted Types - FAILED, TIMED_OUT */
public TaskAttemptEventAttemptFailed(TezTaskAttemptID id,
TaskAttemptEventType type, String diagnostics, TaskAttemptTerminationCause errorCause) {
super(id, type);
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java
new file mode 100644
index 0000000..72e6b07
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.event;
+
+import org.apache.tez.dag.records.TaskAttemptTerminationCause;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+
+public class TaskAttemptEventAttemptKilled extends TaskAttemptEvent
+ implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent {
+
+ private final String diagnostics;
+ private final TaskAttemptTerminationCause errorCause;
+ public TaskAttemptEventAttemptKilled(TezTaskAttemptID id,
+ String diagnostics,
+ TaskAttemptTerminationCause errorCause) {
+ super(id, TaskAttemptEventType.TA_KILLED);
+ this.diagnostics = diagnostics;
+ this.errorCause = errorCause;
+ }
+
+ @Override
+ public String getDiagnosticInfo() {
+ return diagnostics;
+ }
+
+ @Override
+ public TaskAttemptTerminationCause getTerminationCause() {
+ return errorCause;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java
index b7aca36..6d20368 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java
@@ -26,14 +26,15 @@ public enum TaskAttemptEventType {
//Producer:Task, Speculator
TA_SCHEDULE,
-//Producer: TaskAttemptListener
+//Producer: TaskAttemptListener | Vertex after routing events
TA_STARTED_REMOTELY,
TA_STATUS_UPDATE,
TA_DIAGNOSTICS_UPDATE, // REMOVE THIS - UNUSED
TA_DONE,
TA_FAILED,
+ TA_KILLED, // Generated by TaskCommunicators
TA_TIMED_OUT,
-
+
//Producer: Client, Scheduler, On speculation.
TA_KILL_REQUEST,
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 794ef83..10b94f5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.common.TezUtilsInternal;
import org.apache.tez.common.counters.DAGCounter;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.ProcessorDescriptor;
@@ -182,6 +183,11 @@ public class TaskAttemptImpl implements TaskAttempt,
private final StateMachine<TaskAttemptStateInternal, TaskAttemptEventType, TaskAttemptEvent> stateMachine;
+ // TODO TEZ-2003 We may need some additional state management for STATUS_UPDATES, FAILED, KILLED coming in before
+ // TASK_STARTED_REMOTELY. In case of a PUSH it's more intuitive to send TASK_STARTED_REMOTELY after communicating
+ // with the listening service and getting a response, which in turn can trigger STATUS_UPDATES / FAILED / KILLED
+
+ // TA_KILLED handled the same as TA_KILL_REQUEST. Just a different name indicating a request / already killed.
private static StateMachineFactory
<TaskAttemptImpl, TaskAttemptStateInternal, TaskAttemptEventType, TaskAttemptEvent>
stateMachineFactory
@@ -222,6 +228,10 @@ public class TaskAttemptImpl implements TaskAttempt,
new TerminatedBeforeRunningTransition(KILLED_HELPER))
.addTransition(TaskAttemptStateInternal.START_WAIT,
TaskAttemptStateInternal.KILL_IN_PROGRESS,
+ TaskAttemptEventType.TA_KILLED,
+ new TerminatedBeforeRunningTransition(KILLED_HELPER))
+ .addTransition(TaskAttemptStateInternal.START_WAIT,
+ TaskAttemptStateInternal.KILL_IN_PROGRESS,
TaskAttemptEventType.TA_NODE_FAILED,
new NodeFailedBeforeRunningTransition())
.addTransition(TaskAttemptStateInternal.START_WAIT,
@@ -262,6 +272,10 @@ public class TaskAttemptImpl implements TaskAttempt,
new TerminatedWhileRunningTransition(KILLED_HELPER))
.addTransition(TaskAttemptStateInternal.RUNNING,
TaskAttemptStateInternal.KILL_IN_PROGRESS,
+ TaskAttemptEventType.TA_KILLED,
+ new TerminatedWhileRunningTransition(KILLED_HELPER))
+ .addTransition(TaskAttemptStateInternal.RUNNING,
+ TaskAttemptStateInternal.KILL_IN_PROGRESS,
TaskAttemptEventType.TA_NODE_FAILED,
new TerminatedWhileRunningTransition(KILLED_HELPER))
.addTransition(TaskAttemptStateInternal.RUNNING,
@@ -300,6 +314,7 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED,
TaskAttemptEventType.TA_TIMED_OUT,
TaskAttemptEventType.TA_KILL_REQUEST,
+ TaskAttemptEventType.TA_KILLED,
TaskAttemptEventType.TA_NODE_FAILED,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
TaskAttemptEventType.TA_OUTPUT_FAILED))
@@ -321,6 +336,7 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED,
TaskAttemptEventType.TA_TIMED_OUT,
TaskAttemptEventType.TA_KILL_REQUEST,
+ TaskAttemptEventType.TA_KILLED,
TaskAttemptEventType.TA_NODE_FAILED,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
TaskAttemptEventType.TA_OUTPUT_FAILED))
@@ -339,6 +355,7 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED,
TaskAttemptEventType.TA_TIMED_OUT,
TaskAttemptEventType.TA_KILL_REQUEST,
+ TaskAttemptEventType.TA_KILLED,
TaskAttemptEventType.TA_NODE_FAILED,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
TaskAttemptEventType.TA_CONTAINER_TERMINATED,
@@ -358,6 +375,7 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED,
TaskAttemptEventType.TA_TIMED_OUT,
TaskAttemptEventType.TA_KILL_REQUEST,
+ TaskAttemptEventType.TA_KILLED,
TaskAttemptEventType.TA_NODE_FAILED,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
TaskAttemptEventType.TA_CONTAINER_TERMINATED,
@@ -380,6 +398,12 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptStateInternal.SUCCEEDED,
EnumSet.of(TaskAttemptStateInternal.KILLED,
TaskAttemptStateInternal.SUCCEEDED),
+ TaskAttemptEventType.TA_KILLED,
+ new TerminatedAfterSuccessTransition())
+ .addTransition(
+ TaskAttemptStateInternal.SUCCEEDED,
+ EnumSet.of(TaskAttemptStateInternal.KILLED,
+ TaskAttemptStateInternal.SUCCEEDED),
TaskAttemptEventType.TA_NODE_FAILED,
new TerminatedAfterSuccessTransition())
.addTransition(
@@ -427,7 +451,6 @@ public class TaskAttemptImpl implements TaskAttempt,
this.leafVertex = leafVertex;
}
-
@Override
public TezTaskAttemptID getID() {
return attemptId;
@@ -1026,6 +1049,7 @@ public class TaskAttemptImpl implements TaskAttempt,
// Compute node/rack location request even if re-scheduled.
Set<String> racks = new HashSet<String>();
+ // TODO Post TEZ-2003. Allow for a policy in the VMPlugin to define localicty for different attempts.
TaskLocationHint locationHint = ta.getTaskLocationHint();
if (locationHint != null) {
if (locationHint.getRacks() != null) {
@@ -1100,6 +1124,8 @@ public class TaskAttemptImpl implements TaskAttempt,
@Override
public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) {
+ // This transition should not be invoked directly, if a scheduler event has already been sent out.
+ // Sub-classes should be used if a scheduler request has been sent.
ta.setFinishTime();
if (event instanceof DiagnosableEvent) {
@@ -1214,7 +1240,8 @@ public class TaskAttemptImpl implements TaskAttempt,
// Inform the scheduler
if (sendSchedulerEvent()) {
ta.sendEvent(new AMSchedulerEventTAEnded(ta, ta.containerId, helper
- .getTaskAttemptState(), ta.getVertex().getTaskSchedulerIdentifier()));
+ .getTaskAttemptState(), TezUtilsInternal.toTaskAttemptEndReason(ta.terminationCause),
+ ta.getVertex().getTaskSchedulerIdentifier()));
}
}
}
@@ -1296,7 +1323,7 @@ public class TaskAttemptImpl implements TaskAttempt,
// Inform the Scheduler.
ta.sendEvent(new AMSchedulerEventTAEnded(ta, ta.containerId,
- TaskAttemptState.SUCCEEDED, ta.getVertex().getTaskSchedulerIdentifier()));
+ TaskAttemptState.SUCCEEDED, null, ta.getVertex().getTaskSchedulerIdentifier()));
// Inform the task.
ta.sendEvent(new TaskEventTAUpdate(ta.attemptId,
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
index 2ace642..a775948 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
@@ -18,6 +18,7 @@
package org.apache.tez.dag.app.rm;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -27,14 +28,16 @@ public class AMSchedulerEventTAEnded extends AMSchedulerEvent {
private final TaskAttempt attempt;
private final ContainerId containerId;
private final TaskAttemptState state;
+ private final TaskAttemptEndReason taskAttemptEndReason;
private final int schedulerId;
public AMSchedulerEventTAEnded(TaskAttempt attempt, ContainerId containerId,
- TaskAttemptState state, int schedulerId) {
+ TaskAttemptState state, TaskAttemptEndReason taskAttemptEndReason, int schedulerId) {
super(AMSchedulerEventType.S_TA_ENDED);
this.attempt = attempt;
this.containerId = containerId;
this.state = state;
+ this.taskAttemptEndReason = taskAttemptEndReason;
this.schedulerId = schedulerId;
}
@@ -57,4 +60,8 @@ public class AMSchedulerEventTAEnded extends AMSchedulerEvent {
public int getSchedulerId() {
return schedulerId;
}
+
+ public TaskAttemptEndReason getTaskAttemptEndReason() {
+ return taskAttemptEndReason;
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
index 72a074f..a234e07 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
@@ -149,7 +150,7 @@ public class LocalTaskSchedulerService extends TaskSchedulerService {
}
@Override
- public boolean deallocateTask(Object task, boolean taskSucceeded) {
+ public boolean deallocateTask(Object task, boolean taskSucceeded, TaskAttemptEndReason endReason) {
return taskRequestHandler.addDeallocateTaskRequest(task);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 8e5fc71..9f09f68 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -289,7 +289,9 @@ public class TaskSchedulerEventHandler extends AbstractService
private void handleTAUnsuccessfulEnd(AMSchedulerEventTAEnded event) {
TaskAttempt attempt = event.getAttempt();
- boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()].deallocateTask(attempt, false);
+ // Propagate state and failure cause (if any) when informing the scheduler about the de-allocation.
+ boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()]
+ .deallocateTask(attempt, false, event.getTaskAttemptEndReason());
// use stored value of container id in case the scheduler has removed this
// assignment because the task has been deallocated earlier.
// retroactive case
@@ -311,6 +313,7 @@ public class TaskSchedulerEventHandler extends AbstractService
sendEvent(new AMContainerEventStopRequest(attemptContainerId));
// Inform the Node - the task has asked to be STOPPED / has already
// stopped.
+ // AMNodeImpl blacklisting logic does not account for KILLED attempts.
sendEvent(new AMNodeEventTaskAttemptEnded(appContext.getAllContainers().
get(attemptContainerId).getContainer().getNodeId(), attemptContainerId,
attempt.getID(), event.getState() == TaskAttemptState.FAILED));
@@ -332,7 +335,7 @@ public class TaskSchedulerEventHandler extends AbstractService
}
boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()].deallocateTask(attempt,
- true);
+ true, null);
if (!wasContainerAllocated) {
LOG.error("De-allocated successful task: " + attempt.getID()
+ ", but TaskScheduler reported no container assigned to task");
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerService.java
index 48d5455..07dfcd6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerService.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
public abstract class TaskSchedulerService extends AbstractService{
@@ -61,8 +62,9 @@ public abstract class TaskSchedulerService extends AbstractService{
public abstract void allocateTask(Object task, Resource capability,
ContainerId containerId, Priority priority, Object containerSignature,
Object clientCookie);
-
- public abstract boolean deallocateTask(Object task, boolean taskSucceeded);
+
+ /** Plugin writers must ensure to de-allocate a container once it's done, so that it can be collected. */
+ public abstract boolean deallocateTask(Object task, boolean taskSucceeded, TaskAttemptEndReason endReason);
public abstract Object deallocateContainer(ContainerId containerId);
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
index 44f5484..1fc9ac2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
@@ -987,10 +988,13 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
* the task to de-allocate.
* @param taskSucceeded
* specify whether the task succeeded or failed.
+ * @param endReason
+ * reason for the task ending
* @return true if a container is assigned to this task.
*/
@Override
- public boolean deallocateTask(Object task, boolean taskSucceeded) {
+ public boolean deallocateTask(Object task, boolean taskSucceeded,
+ TaskAttemptEndReason endReason) {
Map<CookieContainerRequest, Container> assignedContainers = null;
synchronized (this) {
@@ -1180,7 +1184,7 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
CookieContainerRequest request = entry.getValue();
if (request.getPriority().equals(lowestPriNewContainer.getPriority())) {
LOG.info("Resending request for task again: " + task);
- deallocateTask(task, true);
+ deallocateTask(task, true, null);
allocateTask(task, request.getCapability(),
(request.getNodes() == null ? null :
request.getNodes().toArray(new String[request.getNodes().size()])),
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
index 361565a..f63b636 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
@@ -46,6 +46,7 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.runtime.api.impl.TaskSpec;
import org.junit.Test;
+// TODO TEZ-2003 Rename to TestTezTaskCommunicator
public class TestTaskAttemptListenerImplTezDag {
@Test(timeout = 5000)
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java
new file mode 100644
index 0000000..934543f
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled;
+import org.apache.tez.dag.app.rm.container.AMContainer;
+import org.apache.tez.dag.app.rm.container.AMContainerMap;
+import org.apache.tez.dag.app.rm.container.AMContainerTask;
+import org.apache.tez.dag.records.TaskAttemptTerminationCause;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+// TODO TEZ-2003. Rename to TestTaskAttemptListener | whatever TaskAttemptListener is renamed to.
+public class TestTaskAttemptListenerImplTezDag2 {
+
+ @Test(timeout = 5000)
+ public void testTaskAttemptFailedKilled() {
+ ApplicationId appId = ApplicationId.newInstance(1000, 1);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ Credentials credentials = new Credentials();
+ AppContext appContext = mock(AppContext.class);
+ EventHandler eventHandler = mock(EventHandler.class);
+ DAG dag = mock(DAG.class);
+ AMContainerMap amContainerMap = mock(AMContainerMap.class);
+ Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
+ doReturn(eventHandler).when(appContext).getEventHandler();
+ doReturn(dag).when(appContext).getCurrentDAG();
+ doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+ doReturn(credentials).when(appContext).getAppCredentials();
+ doReturn(appAcls).when(appContext).getApplicationACLs();
+ doReturn(amContainerMap).when(appContext).getAllContainers();
+ NodeId nodeId = NodeId.newInstance("localhost", 0);
+ AMContainer amContainer = mock(AMContainer.class);
+ Container container = mock(Container.class);
+ doReturn(nodeId).when(container).getNodeId();
+ doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
+ doReturn(container).when(amContainer).getContainer();
+
+ TaskAttemptListenerImpTezDag taskAttemptListener =
+ new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
+ mock(ContainerHeartbeatHandler.class), null, null, false);
+
+ TaskSpec taskSpec1 = mock(TaskSpec.class);
+ TezTaskAttemptID taskAttemptId1 = mock(TezTaskAttemptID.class);
+ doReturn(taskAttemptId1).when(taskSpec1).getTaskAttemptID();
+ AMContainerTask amContainerTask1 = new AMContainerTask(taskSpec1, null, null, false, 10);
+
+ TaskSpec taskSpec2 = mock(TaskSpec.class);
+ TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
+ doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
+ AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec2, null, null, false, 10);
+
+ ContainerId containerId1 = createContainerId(appId, 1);
+ taskAttemptListener.registerRunningContainer(containerId1, 0);
+ taskAttemptListener.registerTaskAttempt(amContainerTask1, containerId1, 0);
+ ContainerId containerId2 = createContainerId(appId, 2);
+ taskAttemptListener.registerRunningContainer(containerId2, 0);
+ taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId2, 0);
+
+
+ taskAttemptListener
+ .taskFailed(taskAttemptId1, TaskAttemptEndReason.COMMUNICATION_ERROR, "Diagnostics1");
+ taskAttemptListener
+ .taskKilled(taskAttemptId2, TaskAttemptEndReason.SERVICE_BUSY, "Diagnostics2");
+
+ ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(2)).handle(argumentCaptor.capture());
+ assertTrue(argumentCaptor.getAllValues().get(0) instanceof TaskAttemptEventAttemptFailed);
+ assertTrue(argumentCaptor.getAllValues().get(1) instanceof TaskAttemptEventAttemptKilled);
+ TaskAttemptEventAttemptFailed failedEvent =
+ (TaskAttemptEventAttemptFailed) argumentCaptor.getAllValues().get(0);
+ TaskAttemptEventAttemptKilled killedEvent =
+ (TaskAttemptEventAttemptKilled) argumentCaptor.getAllValues().get(1);
+
+ assertEquals("Diagnostics1", failedEvent.getDiagnosticInfo());
+ assertEquals(TaskAttemptTerminationCause.COMMUNICATION_ERROR,
+ failedEvent.getTerminationCause());
+
+ assertEquals("Diagnostics2", killedEvent.getDiagnosticInfo());
+ assertEquals(TaskAttemptTerminationCause.SERVICE_BUSY, killedEvent.getTerminationCause());
+ // TODO TEZ-2003. Verify unregistration from the registered list
+ }
+
+ private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
+ ContainerId containerId = ContainerId.newInstance(appAttemptId, containerIdx);
+ return containerId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
index c1169ef..d45346a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.isNull;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
@@ -57,6 +58,7 @@ import org.apache.tez.common.MockDNSToSwitchMapping;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TaskLocationHint;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
@@ -92,6 +94,7 @@ import org.apache.tez.runtime.api.impl.OutputSpec;
import org.apache.tez.runtime.api.impl.TaskSpec;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.mockito.internal.matchers.Null;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -223,9 +226,9 @@ public class TestContainerReuse {
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(
- ta11, containerHost1.getId(), TaskAttemptState.SUCCEEDED, 0));
+ ta11, containerHost1.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta11), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler, times(1)).taskAllocated(
eq(ta31), any(Object.class), eq(containerHost1));
verify(rmClient, times(0)).releaseAssignedContainer(
@@ -235,7 +238,7 @@ public class TestContainerReuse {
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta21, containerHost2.getId(),
- TaskAttemptState.SUCCEEDED, 0));
+ TaskAttemptState.SUCCEEDED, null, 0));
long currentTs = System.currentTimeMillis();
Throwable exception = null;
@@ -356,9 +359,9 @@ public class TestContainerReuse {
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta21, containerHost2.getId(),
- TaskAttemptState.SUCCEEDED, 0));
+ TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta21), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta21), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler, times(0)).taskAllocated(
eq(ta31), any(Object.class), eq(containerHost2));
verify(rmClient, times(1)).releaseAssignedContainer(
@@ -459,9 +462,9 @@ public class TestContainerReuse {
verify(taskSchedulerEventHandler).taskAllocated(eq(ta11), any(Object.class), eq(container1));
// Task assigned to container completed successfully. Container should be re-used.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta11), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta12), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
@@ -469,19 +472,19 @@ public class TestContainerReuse {
// Task assigned to container completed successfully.
// Verify reuse across hosts.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta12, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta12, container1.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta12), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta12), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta13), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
eventHandler.reset();
// Verify no re-use if a previous task fails.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta13, container1.getId(), TaskAttemptState.FAILED, 0));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta13, container1.getId(), TaskAttemptState.FAILED, null, 0));
drainableAppCallback.drain();
verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(ta14), any(Object.class), eq(container1));
- verify(taskScheduler).deallocateTask(eq(ta13), eq(false));
+ verify(taskScheduler).deallocateTask(eq(ta13), eq(false), eq((TaskAttemptEndReason)null));
verify(rmClient).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
eventHandler.reset();
@@ -496,9 +499,9 @@ public class TestContainerReuse {
verify(taskSchedulerEventHandler).taskAllocated(eq(ta14), any(Object.class), eq(container2));
// Task assigned to container completed successfully. No pending requests. Container should be released.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta14, container2.getId(), TaskAttemptState.SUCCEEDED, 0));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta14, container2.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta14), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta14), eq(true), eq((TaskAttemptEndReason)null));
verify(rmClient).releaseAssignedContainer(eq(container2.getId()));
eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
eventHandler.reset();
@@ -607,9 +610,9 @@ public class TestContainerReuse {
// First task had profiling on. This container can not be reused further.
taskSchedulerEventHandler.handleEvent(
- new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
+ new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta11), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(ta12), any(Object.class),
eq(container1));
verify(rmClient, times(1)).releaseAssignedContainer(eq(container1.getId()));
@@ -653,9 +656,9 @@ public class TestContainerReuse {
// Verify that the container can not be reused when profiling option is turned on
// Even for 2 tasks having same profiling option can have container reusability.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta13, container2.getId(), TaskAttemptState.SUCCEEDED, 0));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta13, container2.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta13), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta13), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(ta14), any(Object.class),
eq(container2));
verify(rmClient, times(1)).releaseAssignedContainer(eq(container2.getId()));
@@ -698,9 +701,9 @@ public class TestContainerReuse {
verify(taskSchedulerEventHandler).taskAllocated(eq(ta15), any(Object.class), eq(container3));
//Ensure task 6 (of vertex 1) is allocated to same container
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta15, container3.getId(), TaskAttemptState.SUCCEEDED, 0));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta15, container3.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta15), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta15), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta16), any(Object.class), eq(container3));
eventHandler.reset();
@@ -811,9 +814,9 @@ public class TestContainerReuse {
// until delay expires.
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta11, container1.getId(),
- TaskAttemptState.SUCCEEDED, 0));
+ TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta11), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler, times(0)).taskAllocated(
eq(ta12), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
@@ -828,7 +831,7 @@ public class TestContainerReuse {
// TA12 completed.
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta12, container1.getId(),
- TaskAttemptState.SUCCEEDED, 0));
+ TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
LOG.info("Sleeping to ensure that the scheduling loop runs");
Thread.sleep(3000l);
@@ -946,9 +949,9 @@ public class TestContainerReuse {
// Container should be assigned to task21.
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta11, container1.getId(),
- TaskAttemptState.SUCCEEDED, 0));
+ TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta11), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler).taskAllocated(
eq(ta21), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
@@ -956,7 +959,7 @@ public class TestContainerReuse {
// Task 2 completes.
taskSchedulerEventHandler.handleEvent(
new AMSchedulerEventTAEnded(ta21, container1.getId(),
- TaskAttemptState.SUCCEEDED, 0));
+ TaskAttemptState.SUCCEEDED, null, 0));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
LOG.info("Sleeping to ensure that the scheduling loop runs");
@@ -1065,9 +1068,9 @@ public class TestContainerReuse {
assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
// Task assigned to container completed successfully. Container should be re-used.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta111), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta111), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta112), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
@@ -1077,9 +1080,9 @@ public class TestContainerReuse {
// Task assigned to container completed successfully.
// Verify reuse across hosts.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta112), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta112), eq(true), eq((TaskAttemptEndReason)null));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
eventHandler.reset();
@@ -1118,9 +1121,9 @@ public class TestContainerReuse {
assertEquals(2, assignEvent.getRemoteTaskLocalResources().size());
eventHandler.reset();
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta211, container1.getId(), TaskAttemptState.SUCCEEDED, 0));
+ taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta211, container1.getId(), TaskAttemptState.SUCCEEDED, null, 0));
drainableAppCallback.drain();
- verify(taskScheduler).deallocateTask(eq(ta211), eq(true));
+ verify(taskScheduler).deallocateTask(eq(ta211), eq(true), eq((TaskAttemptEndReason)null));
verify(taskSchedulerEventHandler).taskAllocated(eq(ta212), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java
index 25cf4b5..0a642bb 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.rm.TaskSchedulerService.TaskSchedulerAppCallback;
@@ -94,7 +95,7 @@ public class TestLocalTaskSchedulerService {
Task task = mock(Task.class);
taskSchedulerService.allocateTask(task, Resource.newInstance(1024, 1), null, null, Priority.newInstance(1), null, null);
- taskSchedulerService.deallocateTask(task, false);
+ taskSchedulerService.deallocateTask(task, false, null);
// start the RequestHandler, DeallocateTaskRequest has higher priority, so will be processed first
taskSchedulerService.startRequestHandlerThread();
@@ -126,7 +127,7 @@ public class TestLocalTaskSchedulerService {
MockAsyncDelegateRequestHandler requestHandler = taskSchedulerService.getRequestHandler();
requestHandler.drainRequest(1);
- taskSchedulerService.deallocateTask(task, false);
+ taskSchedulerService.deallocateTask(task, false, null);
requestHandler.drainRequest(2);
assertEquals(1, requestHandler.deallocateCount);
assertEquals(1, requestHandler.allocateCount);
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
index dabae67..807e772 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
@@ -172,7 +172,7 @@ public class TestTaskScheduler {
addContainerRequest((CookieContainerRequest) any());
// returned from task requests before allocation happens
- assertFalse(scheduler.deallocateTask(mockTask1, true));
+ assertFalse(scheduler.deallocateTask(mockTask1, true, null));
verify(mockApp, times(0)).containerBeingReleased(any(ContainerId.class));
verify(mockRMClient, times(1)).
removeContainerRequest((CookieContainerRequest) any());
@@ -180,7 +180,7 @@ public class TestTaskScheduler {
releaseAssignedContainer((ContainerId) any());
// deallocating unknown task
- assertFalse(scheduler.deallocateTask(mockTask1, true));
+ assertFalse(scheduler.deallocateTask(mockTask1, true, null));
verify(mockApp, times(0)).containerBeingReleased(any(ContainerId.class));
verify(mockRMClient, times(1)).
removeContainerRequest((CookieContainerRequest) any());
@@ -325,7 +325,7 @@ public class TestTaskScheduler {
verify(mockRMClient).releaseAssignedContainer(mockCId4);
// deallocate allocated task
- assertTrue(scheduler.deallocateTask(mockTask1, true));
+ assertTrue(scheduler.deallocateTask(mockTask1, true, null));
drainableAppCallback.drain();
verify(mockApp).containerBeingReleased(mockCId1);
verify(mockRMClient).releaseAssignedContainer(mockCId1);
@@ -445,7 +445,7 @@ public class TestTaskScheduler {
verify(mockApp, times(4)).taskAllocated(any(), any(), (Container) any());
verify(mockApp).taskAllocated(mockTask4, mockCookie4, mockContainer6);
// deallocate allocated task
- assertTrue(scheduler.deallocateTask(mockTask4, true));
+ assertTrue(scheduler.deallocateTask(mockTask4, true, null));
drainableAppCallback.drain();
verify(mockApp).containerBeingReleased(mockCId6);
verify(mockRMClient).releaseAssignedContainer(mockCId6);
@@ -475,7 +475,7 @@ public class TestTaskScheduler {
removeContainerRequest((CookieContainerRequest) any());
verify(mockRMClient, times(8)).addContainerRequest(
(CookieContainerRequest) any());
- assertFalse(scheduler.deallocateTask(mockTask1, true));
+ assertFalse(scheduler.deallocateTask(mockTask1, true, null));
List<NodeReport> mockUpdatedNodes = mock(List.class);
scheduler.onNodesUpdated(mockUpdatedNodes);
@@ -741,7 +741,7 @@ public class TestTaskScheduler {
verify(mockRMClient).releaseAssignedContainer(mockCId4);
// deallocate allocated task
- assertTrue(scheduler.deallocateTask(mockTask1, true));
+ assertTrue(scheduler.deallocateTask(mockTask1, true, null));
drainableAppCallback.drain();
verify(mockApp).containerBeingReleased(mockCId1);
verify(mockRMClient).releaseAssignedContainer(mockCId1);
@@ -871,7 +871,7 @@ public class TestTaskScheduler {
verify(mockApp, times(4)).taskAllocated(any(), any(), (Container) any());
verify(mockApp).taskAllocated(mockTask4, mockCookie4, mockContainer6);
// deallocate allocated task
- assertTrue(scheduler.deallocateTask(mockTask4, true));
+ assertTrue(scheduler.deallocateTask(mockTask4, true, null));
drainableAppCallback.drain();
verify(mockApp).containerBeingReleased(mockCId6);
verify(mockRMClient).releaseAssignedContainer(mockCId6);
@@ -960,8 +960,8 @@ public class TestTaskScheduler {
// container7 allocated to the task with affinity for it
verify(mockApp).taskAllocated(mockTask6, mockCookie6, mockContainer7);
// deallocate allocated task
- assertTrue(scheduler.deallocateTask(mockTask5, true));
- assertTrue(scheduler.deallocateTask(mockTask6, true));
+ assertTrue(scheduler.deallocateTask(mockTask5, true, null));
+ assertTrue(scheduler.deallocateTask(mockTask6, true, null));
drainableAppCallback.drain();
verify(mockApp).containerBeingReleased(mockCId7);
verify(mockApp).containerBeingReleased(mockCId8);
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
index 5657f86..872d592 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.service.TezTestServiceConfConstants;
@@ -198,7 +199,7 @@ public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
}
@Override
- public boolean deallocateTask(Object task, boolean taskSucceeded) {
+ public boolean deallocateTask(Object task, boolean taskSucceeded, TaskAttemptEndReason endReason) {
ContainerId containerId = runningTasks.remove(task);
if (containerId == null) {
LOG.error("Could not determine ContainerId for task: " + task +
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
index a327caf..e3385a2 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
@@ -19,16 +19,20 @@ import java.nio.ByteBuffer;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.RejectedExecutionException;
import com.google.protobuf.ByteString;
+import com.google.protobuf.ServiceException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.tez.dag.api.TaskAttemptEndReason;
import org.apache.tez.dag.api.TaskCommunicatorContext;
import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
import org.apache.tez.dag.app.TezTestServiceCommunicator;
@@ -83,6 +87,7 @@ public class TezTestServiceTaskCommunicatorImpl extends TezTaskCommunicatorImpl
@Override
public void serviceStop() {
super.serviceStop();
+ this.communicator.stop();
}
@@ -123,13 +128,15 @@ public class TezTestServiceTaskCommunicatorImpl extends TezTaskCommunicatorImpl
throw new RuntimeException("ContainerInfo not found for container: " + containerId +
", while trying to launch task: " + taskSpec.getTaskAttemptID());
}
+ // Have to register this up front right now. Otherwise, it's possible for the task to start
+ // sending out status/DONE/KILLED/FAILED messages before TAImpl knows how to handle them.
+ getTaskCommunicatorContext()
+ .taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId);
communicator.submitWork(requestProto, host, port,
new TezTestServiceCommunicator.ExecuteRequestCallback<SubmitWorkResponseProto>() {
@Override
public void setResponse(SubmitWorkResponseProto response) {
LOG.info("Successfully launched task: " + taskSpec.getTaskAttemptID());
- getTaskCommunicatorContext()
- .taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId);
}
@Override
@@ -137,6 +144,31 @@ public class TezTestServiceTaskCommunicatorImpl extends TezTaskCommunicatorImpl
// TODO Handle this error. This is where an API on the context to indicate failure / rejection comes in.
LOG.info("Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
containerId, t);
+ if (t instanceof ServiceException) {
+ ServiceException se = (ServiceException) t;
+ t = se.getCause();
+ }
+ if (t instanceof RemoteException) {
+ RemoteException re = (RemoteException)t;
+ String message = re.toString();
+ if (message.contains(RejectedExecutionException.class.getName())) {
+ getTaskCommunicatorContext().taskKilled(taskSpec.getTaskAttemptID(),
+ TaskAttemptEndReason.SERVICE_BUSY, "Service Busy");
+ } else {
+ getTaskCommunicatorContext()
+ .taskFailed(taskSpec.getTaskAttemptID(), TaskAttemptEndReason.OTHER,
+ t.toString());
+ }
+ } else {
+ if (t instanceof IOException) {
+ getTaskCommunicatorContext().taskKilled(taskSpec.getTaskAttemptID(),
+ TaskAttemptEndReason.COMMUNICATION_ERROR, "Communication Error");
+ } else {
+ getTaskCommunicatorContext()
+ .taskFailed(taskSpec.getTaskAttemptID(), TaskAttemptEndReason.OTHER,
+ t.getMessage());
+ }
+ }
}
});
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java
index 2bca4ed..28c2286 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java
@@ -16,12 +16,13 @@ package org.apache.tez.service;
import java.io.IOException;
+import org.apache.tez.dag.api.TezException;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
public interface ContainerRunner {
- void queueContainer(RunContainerRequestProto request) throws IOException;
- void submitWork(SubmitWorkRequestProto request) throws IOException;
+ void queueContainer(RunContainerRequestProto request) throws TezException;
+ void submitWork(SubmitWorkRequestProto request) throws TezException;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
index f47bd67..0ac0b33 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
@@ -133,7 +133,10 @@ public class MiniTezTestServiceCluster extends AbstractService {
@Override
public void serviceStop() {
- tezTestService.stop();
+ if (tezTestService != null) {
+ tezTestService.stop();
+ tezTestService = null;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
index 25d6030..379d952 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@@ -58,6 +59,7 @@ import org.apache.tez.common.security.JobTokenIdentifier;
import org.apache.tez.common.security.TokenCache;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
+import org.apache.tez.runtime.api.impl.TaskSpec;
import org.apache.tez.runtime.task.TaskReporter;
import org.apache.tez.runtime.task.TezTaskRunner;
import org.apache.tez.service.ContainerRunner;
@@ -68,14 +70,18 @@ import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl;
import org.apache.tez.runtime.task.TezChild;
import org.apache.tez.runtime.task.TezChild.ContainerExecutionResult;
import org.apache.tez.shufflehandler.ShuffleHandler;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.TaskSpecProto;
import org.apache.tez.util.ProtoConverters;
public class ContainerRunnerImpl extends AbstractService implements ContainerRunner {
private static final Logger LOG = Logger.getLogger(ContainerRunnerImpl.class);
+ public static final String DAG_NAME_INSTRUMENTED_FAILURES = "InstrumentedFailures";
+
private final ListeningExecutorService executorService;
private final AtomicReference<InetSocketAddress> localAddress;
private final String[] localDirsBase;
@@ -146,10 +152,10 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
* Submit a container which is ready for running.
* The regular pull mechanism will be used to fetch work from the AM
* @param request
- * @throws IOException
+ * @throws TezException
*/
@Override
- public void queueContainer(RunContainerRequestProto request) throws IOException {
+ public void queueContainer(RunContainerRequestProto request) throws TezException {
LOG.info("Queuing container for execution: " + request);
Map<String, String> env = new HashMap<String, String>();
@@ -162,7 +168,11 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
for (int i = 0; i < localDirsBase.length; i++) {
localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(),
request.getUser());
- localFs.mkdirs(new Path(localDirs[i]));
+ try {
+ localFs.mkdirs(new Path(localDirs[i]));
+ } catch (IOException e) {
+ throw new TezException(e);
+ }
}
LOG.info("DEBUG: Dirs are: " + Arrays.toString(localDirs));
@@ -175,7 +185,11 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
DataInputBuffer dib = new DataInputBuffer();
byte[] tokenBytes = request.getCredentialsBinary().toByteArray();
dib.reset(tokenBytes, tokenBytes.length);
- credentials.readTokenStorageStream(dib);
+ try {
+ credentials.readTokenStorageStream(dib);
+ } catch (IOException e) {
+ throw new TezException(e);
+ }
Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
@@ -197,13 +211,14 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
* This is intended for a task push from the AM
*
* @param request
- * @throws IOException
+ * @throws org.apache.tez.dag.api.TezException
*/
@Override
- public void submitWork(SubmitWorkRequestProto request) throws
- IOException {
+ public void submitWork(SubmitWorkRequestProto request) throws TezException {
LOG.info("Queuing work for execution: " + request);
+ checkAndThrowExceptionForTests(request);
+
Map<String, String> env = new HashMap<String, String>();
env.putAll(localEnv);
env.put(ApplicationConstants.Environment.USER.name(), request.getUser());
@@ -214,7 +229,11 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
for (int i = 0; i < localDirsBase.length; i++) {
localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(),
request.getUser());
- localFs.mkdirs(new Path(localDirs[i]));
+ try {
+ localFs.mkdirs(new Path(localDirs[i]));
+ } catch (IOException e) {
+ throw new TezException(e);
+ }
}
if (LOG.isDebugEnabled()) {
LOG.debug("Dirs are: " + Arrays.toString(localDirs));
@@ -228,7 +247,11 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
DataInputBuffer dib = new DataInputBuffer();
byte[] tokenBytes = request.getCredentialsBinary().toByteArray();
dib.reset(tokenBytes, tokenBytes.length);
- credentials.readTokenStorageStream(dib);
+ try {
+ credentials.readTokenStorageStream(dib);
+ } catch (IOException e) {
+ throw new TezException(e);
+ }
Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
@@ -509,4 +532,23 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
}
}
+
+ private void checkAndThrowExceptionForTests(SubmitWorkRequestProto request) throws TezException {
+ if (!request.getTaskSpec().getDagName().equals(DAG_NAME_INSTRUMENTED_FAILURES)) {
+ return;
+ }
+
+ TaskSpec taskSpec = ProtoConverters.getTaskSpecfromProto(request.getTaskSpec());
+ if (taskSpec.getTaskAttemptID().getTaskID().getId() == 0 &&
+ taskSpec.getTaskAttemptID().getId() == 0) {
+ LOG.info("Simulating Rejected work");
+ throw new RejectedExecutionException(
+ "Simulating Rejected work for taskAttemptId=" + taskSpec.getTaskAttemptID());
+ } else if (taskSpec.getTaskAttemptID().getTaskID().getId() == 1 &&
+ taskSpec.getTaskAttemptID().getId() == 0) {
+ LOG.info("Simulating Task Setup Failure during launch");
+ throw new TezException("Simulating Task Setup Failure during launch for taskAttemptId=" +
+ taskSpec.getTaskAttemptID());
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
index 012e352..855f1b0 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
@@ -14,7 +14,6 @@
package org.apache.tez.service.impl;
-import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicInteger;
@@ -25,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Logger;
+import org.apache.tez.dag.api.TezException;
import org.apache.tez.service.ContainerRunner;
import org.apache.tez.shufflehandler.ShuffleHandler;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
@@ -108,14 +108,14 @@ public class TezTestService extends AbstractService implements ContainerRunner {
@Override
- public void queueContainer(RunContainerRequestProto request) throws IOException {
+ public void queueContainer(RunContainerRequestProto request) throws TezException {
numSubmissions.incrementAndGet();
containerRunner.queueContainer(request);
}
@Override
public void submitWork(TezTestServiceProtocolProtos.SubmitWorkRequestProto request) throws
- IOException {
+ TezException {
numSubmissions.incrementAndGet();
containerRunner.submitWork(request);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
index d7f8444..39d7156 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
@@ -30,11 +30,13 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.service.AbstractService;
+import org.apache.tez.dag.api.TezException;
import org.apache.tez.service.ContainerRunner;
import org.apache.tez.service.TezTestServiceProtocolBlockingPB;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto;
public class TezTestServiceProtocolServerImpl extends AbstractService
@@ -61,20 +63,20 @@ public class TezTestServiceProtocolServerImpl extends AbstractService
LOG.info("Received request: " + request);
try {
containerRunner.queueContainer(request);
- } catch (IOException e) {
+ } catch (TezException e) {
throw new ServiceException(e);
}
return RunContainerResponseProto.getDefaultInstance();
}
@Override
- public SubmitWorkResponseProto submitWork(RpcController controller, TezTestServiceProtocolProtos.SubmitWorkRequestProto request) throws
+ public SubmitWorkResponseProto submitWork(RpcController controller, SubmitWorkRequestProto request) throws
ServiceException {
LOG.info("Received submitWork request: " + request);
try {
containerRunner.submitWork(request);
- } catch (IOException e) {
- e.printStackTrace();
+ } catch (TezException e) {
+ throw new ServiceException(e);
}
return SubmitWorkResponseProto.getDefaultInstance();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/e901aeb6/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
index 0ec972b..b6a166d 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
@@ -27,16 +27,23 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.ProcessorDescriptor;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.app.launcher.TezTestServiceNoOpContainerLauncher;
import org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerService;
import org.apache.tez.dag.app.taskcomm.TezTestServiceTaskCommunicatorImpl;
import org.apache.tez.examples.HashJoinExample;
import org.apache.tez.examples.JoinDataGen;
import org.apache.tez.examples.JoinValidateConfigured;
+import org.apache.tez.runtime.library.processor.SleepProcessor;
import org.apache.tez.service.MiniTezTestServiceCluster;
+import org.apache.tez.service.impl.ContainerRunnerImpl;
import org.apache.tez.test.MiniTezCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -283,6 +290,28 @@ public class TestExternalTezServices {
PROPS_IN_AM, PROPS_REGULAR_CONTAINERS);
}
+ @Test(timeout = 60000)
+ public void testErrorPropagation() throws TezException, InterruptedException, IOException {
+ runExceptionSimulation();
+ }
+
+
+
+ private void runExceptionSimulation() throws IOException, TezException, InterruptedException {
+ DAG dag = DAG.create(ContainerRunnerImpl.DAG_NAME_INSTRUMENTED_FAILURES);
+ Vertex v =Vertex.create("Vertex1", ProcessorDescriptor.create(SleepProcessor.class.getName()),
+ 3);
+ for (Map.Entry<String, String> prop : PROPS_EXT_SERVICE_PUSH.entrySet()) {
+ v.setConf(prop.getKey(), prop.getValue());
+ }
+ dag.addVertex(v);
+ DAGClient dagClient = sharedTezClient.submitDAG(dag);
+ DAGStatus dagStatus = dagClient.waitForCompletion();
+ assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
+ assertEquals(1, dagStatus.getDAGProgress().getFailedTaskAttemptCount());
+ assertEquals(1, dagStatus.getDAGProgress().getKilledTaskAttemptCount());
+
+ }
private void runJoinValidate(String name, int extExpectedCount, Map<String, String> lhsProps,
Map<String, String> rhsProps,
[03/50] [abbrv] tez git commit: TEZ-2318. Tez UI: source and sink
page is broken as they are not populated. (Sreenath Somarajapuram via
pramachandran)
Posted by ss...@apache.org.
TEZ-2318. Tez UI: source and sink page is broken as they are not populated. (Sreenath Somarajapuram via pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/fa72bdc7
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/fa72bdc7
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/fa72bdc7
Branch: refs/heads/TEZ-2003
Commit: fa72bdc7f53dd9e9d0f74fef6b17af679ae3cc83
Parents: 6019bbc
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Tue Apr 14 20:07:57 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Tue Apr 14 20:07:57 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../webapp/app/scripts/models/TimelineRestAdapter.js | 14 ++++++++++----
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/fa72bdc7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5fe83cb..b47cf7d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2318. Tez UI: source and sink page is broken as they are not populated.
TEZ-2016. Tez UI: Dag View Fit and Finish
TEZ-2252. Tez UI: in graphical view some of the sinks are hidden as they overlap
TEZ-2275. Tez UI: enable faster loading and caching of data in tables
http://git-wip-us.apache.org/repos/asf/tez/blob/fa72bdc7/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js b/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js
index a54ded2..9c84771 100644
--- a/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js
+++ b/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js
@@ -245,12 +245,13 @@ App.VertexSerializer = App.TimelineSerializer.extend({
normalizePayload: function(rawPayload, property) {
var pluralizedPoperty,
+ normalizedPayload,
n;
property = property || 'vertex',
pluralizedPoperty = property.pluralize();;
if (!!rawPayload[pluralizedPoperty]) {
- var normalizedPayload = {
+ normalizedPayload = {
inputs: [],
outputs: [],
};
@@ -268,9 +269,14 @@ App.VertexSerializer = App.TimelineSerializer.extend({
return normalizedPayload;
} else {
- n = {};
- n[property] = this._normalizeSingleVertexPayload(rawPayload[property]).vertex;
- return n;
+ n = this._normalizeSingleVertexPayload(rawPayload[property]);
+ normalizedPayload = {
+ inputs : n.inputs,
+ outputs: n.outputs
+ };
+ normalizedPayload[property] = n.vertex;
+
+ return normalizedPayload;
}
},
[50/50] [abbrv] tez git commit: TEZ-2347. Expose additional
information in TaskCommunicatorContext. (sseth)
Posted by ss...@apache.org.
TEZ-2347. Expose additional information in TaskCommunicatorContext. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/51b9db1a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/51b9db1a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/51b9db1a
Branch: refs/heads/TEZ-2003
Commit: 51b9db1a0401d037d454c3f7be48523634b1dae7
Parents: 74c05f6
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Apr 20 13:17:31 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:48:02 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../tez/dag/api/TaskCommunicatorContext.java | 50 ++++++++++++++++++++
.../dag/app/TaskCommunicatorContextImpl.java | 50 ++++++++++++++++++++
.../java/org/apache/tez/dag/app/dag/DAG.java | 2 +
.../java/org/apache/tez/dag/app/dag/Task.java | 1 +
.../org/apache/tez/dag/app/dag/TaskAttempt.java | 6 +++
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 10 ++++
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 12 +++++
.../apache/tez/dag/app/dag/impl/TaskImpl.java | 13 ++++-
9 files changed, 144 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/51b9db1a/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index ca5225e..7c13110 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -16,5 +16,6 @@ ALL CHANGES:
TEZ-2284. Separate TaskReporter into an interface.
TEZ-2285. Allow TaskCommunicators to indicate task/container liveness.
TEZ-2302. Allow TaskCommunicators to subscribe for Vertex updates.
+ TEZ-2347. Expose additional information in TaskCommunicatorContext.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/51b9db1a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
index 19caed9..56345ab 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
@@ -16,6 +16,7 @@ package org.apache.tez.dag.api;
import javax.annotation.Nullable;
import java.io.IOException;
+import java.util.Collection;
import java.util.Set;
import org.apache.hadoop.security.Credentials;
@@ -71,4 +72,53 @@ public interface TaskCommunicatorContext {
// TODO TEZ-2003 API. Should a method exist for task succeeded.
// TODO Eventually Add methods to report availability stats to the scheduler.
+
+ /**
+ * Get the name of the currently executing dag
+ * @return the name of the currently executing dag
+ */
+ String getCurretnDagName();
+
+ /**
+ * Get the name of the Input vertices for the specified vertex.
+ * Root Inputs are not returned.
+ * @param vertexName the vertex for which source vertex names will be returned
+ * @return an Iterable containing the list of input vertices for the specified vertex
+ */
+ Iterable<String> getInputVertexNames(String vertexName);
+
+ /**
+ * Get the total number of tasks in the given vertex
+ * @param vertexName
+ * @return total number of tasks in this vertex
+ */
+ int getVertexTotalTaskCount(String vertexName);
+
+ /**
+ * Get the number of completed tasks for a given vertex
+ * @param vertexName the vertex name
+ * @return the number of completed tasks for the vertex
+ */
+ int getVertexCompletedTaskCount(String vertexName);
+
+ /**
+ * Get the number of running tasks for a given vertex
+ * @param vertexName the vertex name
+ * @return the number of running tasks for the vertex
+ */
+ int getVertexRunningTaskCount(String vertexName);
+
+ /**
+ * Get the start time for the first attempt of the specified task
+ * @param vertexName the vertex to which the task belongs
+ * @param taskIndex the index of the task
+ * @return the start time for the first attempt of the task
+ */
+ long getFirstAttemptStartTime(String vertexName, int taskIndex);
+
+ /**
+ * Get the start time for the currently executing DAG
+ * @return time when the current dag started executing
+ */
+ long getDagStartTime();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/51b9db1a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
index 3714c3c..4cb0c93 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
@@ -18,7 +18,9 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Set;
+import com.google.common.base.Function;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -31,6 +33,7 @@ import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.event.VertexState;
import org.apache.tez.dag.api.event.VertexStateUpdate;
+import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.VertexStateUpdateListener;
import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -111,6 +114,53 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
context.getCurrentDAG().getStateChangeNotifier().registerForVertexUpdates(vertexName, stateSet, this);
}
+ @Override
+ public String getCurretnDagName() {
+ return context.getCurrentDAG().getName();
+ }
+
+ @Override
+ public Iterable<String> getInputVertexNames(String vertexName) {
+ Preconditions.checkNotNull(vertexName, "VertexName cannot be null: " + vertexName);
+ Vertex vertex = context.getCurrentDAG().getVertex(vertexName);
+ Set<Vertex> sources = vertex.getInputVertices().keySet();
+ return Iterables.transform(sources, new Function<Vertex, String>() {
+ @Override
+ public String apply(@Nullable Vertex input) {
+ return input.getName();
+ }
+ });
+ }
+
+ @Override
+ public int getVertexTotalTaskCount(String vertexName) {
+ Preconditions.checkArgument(vertexName != null, "VertexName must be specified");
+ return context.getCurrentDAG().getVertex(vertexName).getTotalTasks();
+ }
+
+ @Override
+ public int getVertexCompletedTaskCount(String vertexName) {
+ Preconditions.checkArgument(vertexName != null, "VertexName must be specified");
+ return context.getCurrentDAG().getVertex(vertexName).getCompletedTasks();
+ }
+
+ @Override
+ public int getVertexRunningTaskCount(String vertexName) {
+ Preconditions.checkArgument(vertexName != null, "VertexName must be specified");
+ return context.getCurrentDAG().getVertex(vertexName).getRunningTasks();
+ }
+
+ @Override
+ public long getFirstAttemptStartTime(String vertexName, int taskIndex) {
+ Preconditions.checkArgument(vertexName != null, "VertexName must be specified");
+ Preconditions.checkArgument(taskIndex >=0, "TaskIndex must be > 0");
+ return context.getCurrentDAG().getVertex(vertexName).getTask(taskIndex).getFirstAttemptStartTime();
+ }
+
+ @Override
+ public long getDagStartTime() {
+ return context.getCurrentDAG().getStartTime();
+ }
@Override
public void onStateUpdated(VertexStateUpdate event) {
http://git-wip-us.apache.org/repos/asf/tez/blob/51b9db1a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index 6d6872b..458362f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -94,6 +94,8 @@ public interface DAG {
Map<String, TezVertexID> getVertexNameIDMapping();
+ long getStartTime();
+
StateChangeNotifier getStateChangeNotifier();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/51b9db1a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
index b798fce..3af14b5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
@@ -66,4 +66,5 @@ public interface Task {
public void registerTezEvent(TezEvent tezEvent);
+ long getFirstAttemptStartTime();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/51b9db1a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
index 3f60a4e..4e4cb1b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
@@ -122,6 +122,12 @@ public interface TaskAttempt {
*/
long getLaunchTime();
+ /**
+ * Get the time at which this attempt was scheduled
+ * @return the time at which this attempt was scheduled, 0 if it hasn't been scheduled yet
+ */
+ long getScheduleTime();
+
/**
* @return attempt's finish time. If attempt is not finished
* yet, returns 0.
http://git-wip-us.apache.org/repos/asf/tez/blob/51b9db1a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 66809d4..128b4fb 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -681,6 +681,16 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
@Override
+ public long getStartTime() {
+ readLock.lock();
+ try {
+ return this.startTime;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
public StateChangeNotifier getStateChangeNotifier() {
return entityUpdateTracker;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/51b9db1a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 10b94f5..d985213 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -135,6 +135,7 @@ public class TaskAttemptImpl implements TaskAttempt,
protected final AppContext appContext;
private final TaskHeartbeatHandler taskHeartbeatHandler;
private long launchTime = 0;
+ private long scheduleTime = 0;
private long finishTime = 0;
private String trackerName;
private int httpPort;
@@ -664,6 +665,16 @@ public class TaskAttemptImpl implements TaskAttempt,
}
@Override
+ public long getScheduleTime() {
+ readLock.lock();
+ try {
+ return scheduleTime;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
public long getFinishTime() {
readLock.lock();
try {
@@ -1026,6 +1037,7 @@ public class TaskAttemptImpl implements TaskAttempt,
public TaskAttemptStateInternal transition(TaskAttemptImpl ta, TaskAttemptEvent event) {
TaskAttemptEventSchedule scheduleEvent = (TaskAttemptEventSchedule) event;
+ ta.scheduleTime = ta.clock.getTime();
// TODO Creating the remote task here may not be required in case of
// recovery.
http://git-wip-us.apache.org/repos/asf/tez/blob/51b9db1a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index a1eed07..07b57cc 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -1470,7 +1470,18 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
this.writeLock.unlock();
}
}
-
+
+ @Override
+ public long getFirstAttemptStartTime() {
+ readLock.lock();
+ try {
+ // The first attempt will always have an index of 0.
+ return getAttempt(TezTaskAttemptID.getInstance(getTaskId(), 0)).getScheduleTime();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
@Private
@VisibleForTesting
public List<TezEvent> getTaskEvents() {
[31/50] [abbrv] tez git commit: TEZ-2019. Temporarily allow the
scheduler and launcher to be specified via configuration. (sseth)
Posted by ss...@apache.org.
TEZ-2019. Temporarily allow the scheduler and launcher to be specified
via configuration. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/82ba10c1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/82ba10c1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/82ba10c1
Branch: refs/heads/TEZ-2003
Commit: 82ba10c1f0f9385548726a79b2d0f948b2da0e0e
Parents: ec7b481
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Jan 30 16:02:32 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:44:31 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 4 +++
.../apache/tez/dag/api/TezConfiguration.java | 6 ++++
.../org/apache/tez/dag/app/DAGAppMaster.java | 30 ++++++++++++++++-
.../dag/app/rm/TaskSchedulerEventHandler.java | 34 ++++++++++++++++++--
.../org/apache/tez/runtime/task/TezChild.java | 3 +-
5 files changed, 73 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/82ba10c1/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
new file mode 100644
index 0000000..1822fcb
--- /dev/null
+++ b/TEZ-2003-CHANGES.txt
@@ -0,0 +1,4 @@
+ALL CHANGES:
+ TEZ-2019. Temporarily allow the scheduler and launcher to be specified via configuration.
+
+INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/82ba10c1/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 7a49ee5..3509509 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1136,6 +1136,12 @@ public class TezConfiguration extends Configuration {
+ "tez-ui.webservice.enable";
public static final boolean TEZ_AM_WEBSERVICE_ENABLE_DEFAULT = true;
+ @ConfigurationScope(Scope.VERTEX)
+ public static final String TEZ_AM_CONTAINER_LAUNCHER_CLASS = TEZ_AM_PREFIX + "container-launcher.class";
+ @ConfigurationScope(Scope.VERTEX)
+ public static final String TEZ_AM_TASK_SCHEDULER_CLASS = TEZ_AM_PREFIX + "task-scheduler.class";
+
+
// TODO only validate property here, value can also be validated if necessary
public static void validateProperty(String property, Scope usedScope) {
Scope validScope = PropertyScope.get(property);
http://git-wip-us.apache.org/repos/asf/tez/blob/82ba10c1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 8a914f6..a6d6c69 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -25,6 +25,8 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@@ -93,6 +95,7 @@ import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.tez.common.AsyncDispatcher;
import org.apache.tez.common.GcTimeUpdater;
+import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezConverterUtils;
import org.apache.tez.common.TezUtilsInternal;
@@ -1022,9 +1025,34 @@ public class DAGAppMaster extends AbstractService {
protected ContainerLauncher
createContainerLauncher(final AppContext context) throws UnknownHostException {
if(isLocal){
+ LOG.info("Creating LocalContainerLauncher");
return new LocalContainerLauncher(context, taskAttemptListener, workingDirectory);
} else {
- return new ContainerLauncherImpl(context);
+ // TODO: Temporary reflection with specific parameters until a clean interface is defined.
+ String containerLauncherClassName = getConfig().get(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHER_CLASS);
+ if (containerLauncherClassName == null) {
+ LOG.info("Creating Default Container Launcher");
+ return new ContainerLauncherImpl(context);
+ } else {
+ LOG.info("Creating container launcher : " + containerLauncherClassName);
+ Class<? extends ContainerLauncher> containerLauncherClazz = (Class<? extends ContainerLauncher>) ReflectionUtils.getClazz(
+ containerLauncherClassName);
+ try {
+ Constructor<? extends ContainerLauncher> ctor = containerLauncherClazz
+ .getConstructor(AppContext.class, Configuration.class, TaskAttemptListener.class);
+ ctor.setAccessible(true);
+ ContainerLauncher instance = ctor.newInstance(context, getConfig(), taskAttemptListener);
+ return instance;
+ } catch (NoSuchMethodException e) {
+ throw new TezUncheckedException(e);
+ } catch (InvocationTargetException e) {
+ throw new TezUncheckedException(e);
+ } catch (InstantiationException e) {
+ throw new TezUncheckedException(e);
+ } catch (IllegalAccessException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/82ba10c1/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 19db660..62f82db 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -18,6 +18,8 @@
package org.apache.tez.dag.app.rm;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.List;
@@ -42,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.TaskLocationHint;
@@ -329,12 +332,39 @@ public class TaskSchedulerEventHandler extends AbstractService
boolean isLocal = getConfig().getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
if (isLocal) {
+ LOG.info("Using TaskScheduler: LocalTaskSchedulerService");
return new LocalTaskSchedulerService(this, this.containerSignatureMatcher,
host, port, trackingUrl, appContext);
}
else {
- return new YarnTaskSchedulerService(this, this.containerSignatureMatcher,
- host, port, trackingUrl, appContext);
+ String schedulerClassName = getConfig().get(TezConfiguration.TEZ_AM_TASK_SCHEDULER_CLASS);
+ if (schedulerClassName == null) {
+ LOG.info("Using TaskScheduler: YarnTaskSchedulerService");
+ return new YarnTaskSchedulerService(this, this.containerSignatureMatcher,
+ host, port, trackingUrl, appContext);
+ } else {
+ LOG.info("Using custom TaskScheduler: " + schedulerClassName);
+ // TODO Temporary reflection with specific parameters. Remove once there is a clean interface.
+ Class<? extends TaskSchedulerService> taskSchedulerClazz =
+ (Class<? extends TaskSchedulerService>) ReflectionUtils.getClazz(schedulerClassName);
+ try {
+ Constructor<? extends TaskSchedulerService> ctor = taskSchedulerClazz
+ .getConstructor(TaskSchedulerAppCallback.class, AppContext.class, String.class,
+ Integer.class, String.class, Configuration.class);
+ ctor.setAccessible(true);
+ TaskSchedulerService taskSchedulerService =
+ ctor.newInstance(this, appContext, host, port, trackingUrl, getConfig());
+ return taskSchedulerService;
+ } catch (NoSuchMethodException e) {
+ throw new TezUncheckedException(e);
+ } catch (InvocationTargetException e) {
+ throw new TezUncheckedException(e);
+ } catch (InstantiationException e) {
+ throw new TezUncheckedException(e);
+ } catch (IllegalAccessException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/82ba10c1/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index bfec349..fd55992 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -355,7 +355,8 @@ public class TezChild {
}
if (!isLocal) {
RPC.stopProxy(umbilical);
- LogManager.shutdown();
+ // TODO Temporary change. Revert. Ideally, move this over to the main method in TezChild if possible.
+// LogManager.shutdown();
}
}
}
[32/50] [abbrv] tez git commit: TEZ-2090. Add tests for jobs running
in external services. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
new file mode 100644
index 0000000..a93c1a4
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.tests;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.app.launcher.TezTestServiceNoOpContainerLauncher;
+import org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerService;
+import org.apache.tez.dag.app.taskcomm.TezTestServiceTaskCommunicatorImpl;
+import org.apache.tez.examples.HashJoinExample;
+import org.apache.tez.examples.JoinDataGen;
+import org.apache.tez.examples.JoinValidate;
+import org.apache.tez.service.MiniTezTestServiceCluster;
+import org.apache.tez.test.MiniTezCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestExternalTezServices {
+
+ private static final Log LOG = LogFactory.getLog(TestExternalTezServices.class);
+
+ private static MiniTezCluster tezCluster;
+ private static MiniDFSCluster dfsCluster;
+ private static MiniTezTestServiceCluster tezTestServiceCluster;
+
+ private static Configuration clusterConf = new Configuration();
+ private static Configuration confForJobs;
+
+ private static FileSystem remoteFs;
+ private static FileSystem localFs;
+
+ private static TezClient sharedTezClient;
+
+ private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + TestExternalTezServices.class.getName()
+ + "-tmpDir";
+
+ @BeforeClass
+ public static void setup() throws IOException, TezException, InterruptedException {
+
+ localFs = FileSystem.getLocal(clusterConf);
+
+ try {
+ clusterConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
+ dfsCluster =
+ new MiniDFSCluster.Builder(clusterConf).numDataNodes(1).format(true).racks(null).build();
+ remoteFs = dfsCluster.getFileSystem();
+ LOG.info("MiniDFSCluster started");
+ } catch (IOException io) {
+ throw new RuntimeException("problem starting mini dfs cluster", io);
+ }
+
+ tezCluster = new MiniTezCluster(TestExternalTezServices.class.getName(), 1, 1, 1);
+ Configuration conf = new Configuration();
+ conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
+ tezCluster.init(conf);
+ tezCluster.start();
+ LOG.info("MiniTezCluster started");
+
+ clusterConf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
+ for (Map.Entry<String, String> entry : tezCluster.getConfig()) {
+ clusterConf.set(entry.getKey(), entry.getValue());
+ }
+ long jvmMax = Runtime.getRuntime().maxMemory();
+
+ tezTestServiceCluster = MiniTezTestServiceCluster
+ .create(TestExternalTezServices.class.getSimpleName(), 3, ((long) (jvmMax * 0.5d)), 1);
+ tezTestServiceCluster.init(clusterConf);
+ tezTestServiceCluster.start();
+ LOG.info("MiniTezTestServer started");
+
+ confForJobs = new Configuration(clusterConf);
+ for (Map.Entry<String, String> entry : tezTestServiceCluster
+ .getClusterSpecificConfiguration()) {
+ confForJobs.set(entry.getKey(), entry.getValue());
+ }
+
+ // TODO TEZ-2003 Once per vertex configuration is possible, run separate tests for push vs pull (regular threaded execution)
+
+ Path stagingDirPath = new Path("/tmp/tez-staging-dir");
+ remoteFs.mkdirs(stagingDirPath);
+ // This is currently configured to push tasks into the Service, and then use the standard RPC
+ confForJobs.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
+ confForJobs.set(TezConfiguration.TEZ_AM_TASK_SCHEDULER_CLASS,
+ TezTestServiceTaskSchedulerService.class.getName());
+ confForJobs.set(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHER_CLASS,
+ TezTestServiceNoOpContainerLauncher.class.getName());
+ confForJobs.set(TezConfiguration.TEZ_AM_TASK_COMMUNICATOR_CLASS,
+ TezTestServiceTaskCommunicatorImpl.class.getName());
+
+ TezConfiguration tezConf = new TezConfiguration(confForJobs);
+
+ sharedTezClient = TezClient.create(TestExternalTezServices.class.getSimpleName() + "_session",
+ tezConf, true);
+ sharedTezClient.start();
+ LOG.info("Shared TezSession started");
+ sharedTezClient.waitTillReady();
+ LOG.info("Shared TezSession ready for submission");
+
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException, TezException {
+ if (sharedTezClient != null) {
+ sharedTezClient.stop();
+ sharedTezClient = null;
+ }
+
+ if (tezTestServiceCluster != null) {
+ tezTestServiceCluster.stop();
+ tezTestServiceCluster = null;
+ }
+
+ if (tezCluster != null) {
+ tezCluster.stop();
+ tezCluster = null;
+ }
+ if (dfsCluster != null) {
+ dfsCluster.shutdown();
+ dfsCluster = null;
+ }
+ // TODO Add cleanup code.
+ }
+
+
+ @Test(timeout = 60000)
+ public void test1() throws Exception {
+ Path testDir = new Path("/tmp/testHashJoinExample");
+
+ remoteFs.mkdirs(testDir);
+
+ Path dataPath1 = new Path(testDir, "inPath1");
+ Path dataPath2 = new Path(testDir, "inPath2");
+ Path expectedOutputPath = new Path(testDir, "expectedOutputPath");
+ Path outPath = new Path(testDir, "outPath");
+
+ TezConfiguration tezConf = new TezConfiguration(confForJobs);
+
+ JoinDataGen dataGen = new JoinDataGen();
+ String[] dataGenArgs = new String[]{
+ dataPath1.toString(), "1048576", dataPath2.toString(), "524288",
+ expectedOutputPath.toString(), "2"};
+ assertEquals(0, dataGen.run(tezConf, dataGenArgs, sharedTezClient));
+
+ HashJoinExample joinExample = new HashJoinExample();
+ String[] args = new String[]{
+ dataPath1.toString(), dataPath2.toString(), "2", outPath.toString()};
+ assertEquals(0, joinExample.run(tezConf, args, sharedTezClient));
+
+ JoinValidate joinValidate = new JoinValidate();
+ String[] validateArgs = new String[]{
+ expectedOutputPath.toString(), outPath.toString(), "3"};
+ assertEquals(0, joinValidate.run(tezConf, validateArgs, sharedTezClient));
+
+ // Ensure this was actually submitted to the external cluster
+ assertTrue(tezTestServiceCluster.getNumSubmissions() > 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java b/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java
new file mode 100644
index 0000000..60ebc53
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.tez.dag.api.DagTypeConverters;
+import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.OutputDescriptor;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.GroupInputSpec;
+import org.apache.tez.runtime.api.impl.InputSpec;
+import org.apache.tez.runtime.api.impl.OutputSpec;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.GroupInputSpecProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.IOSpecProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.TaskSpecProto;
+
+public class ProtoConverters {
+
+ public static TaskSpec getTaskSpecfromProto(TaskSpecProto taskSpecProto) {
+ TezTaskAttemptID taskAttemptID =
+ TezTaskAttemptID.fromString(taskSpecProto.getTaskAttemptIdString());
+
+ ProcessorDescriptor processorDescriptor = null;
+ if (taskSpecProto.hasProcessorDescriptor()) {
+ processorDescriptor = DagTypeConverters
+ .convertProcessorDescriptorFromDAGPlan(taskSpecProto.getProcessorDescriptor());
+ }
+
+ List<InputSpec> inputSpecList = new ArrayList<InputSpec>(taskSpecProto.getInputSpecsCount());
+ if (taskSpecProto.getInputSpecsCount() > 0) {
+ for (IOSpecProto inputSpecProto : taskSpecProto.getInputSpecsList()) {
+ inputSpecList.add(getInputSpecFromProto(inputSpecProto));
+ }
+ }
+
+ List<OutputSpec> outputSpecList =
+ new ArrayList<OutputSpec>(taskSpecProto.getOutputSpecsCount());
+ if (taskSpecProto.getOutputSpecsCount() > 0) {
+ for (IOSpecProto outputSpecProto : taskSpecProto.getOutputSpecsList()) {
+ outputSpecList.add(getOutputSpecFromProto(outputSpecProto));
+ }
+ }
+
+ List<GroupInputSpec> groupInputSpecs =
+ new ArrayList<GroupInputSpec>(taskSpecProto.getGroupedInputSpecsCount());
+ if (taskSpecProto.getGroupedInputSpecsCount() > 0) {
+ for (GroupInputSpecProto groupInputSpecProto : taskSpecProto.getGroupedInputSpecsList()) {
+ groupInputSpecs.add(getGroupInputSpecFromProto(groupInputSpecProto));
+ }
+ }
+
+ TaskSpec taskSpec =
+ new TaskSpec(taskAttemptID, taskSpecProto.getDagName(), taskSpecProto.getVertexName(),
+ taskSpecProto.getVertexParallelism(), processorDescriptor, inputSpecList,
+ outputSpecList, groupInputSpecs);
+ return taskSpec;
+ }
+
+ public static TaskSpecProto convertTaskSpecToProto(TaskSpec taskSpec) {
+ TaskSpecProto.Builder builder = TaskSpecProto.newBuilder();
+ builder.setTaskAttemptIdString(taskSpec.getTaskAttemptID().toString());
+ builder.setDagName(taskSpec.getDAGName());
+ builder.setVertexName(taskSpec.getVertexName());
+ builder.setVertexParallelism(taskSpec.getVertexParallelism());
+
+ if (taskSpec.getProcessorDescriptor() != null) {
+ builder.setProcessorDescriptor(
+ DagTypeConverters.convertToDAGPlan(taskSpec.getProcessorDescriptor()));
+ }
+
+ if (taskSpec.getInputs() != null && !taskSpec.getInputs().isEmpty()) {
+ for (InputSpec inputSpec : taskSpec.getInputs()) {
+ builder.addInputSpecs(convertInputSpecToProto(inputSpec));
+ }
+ }
+
+ if (taskSpec.getOutputs() != null && !taskSpec.getOutputs().isEmpty()) {
+ for (OutputSpec outputSpec : taskSpec.getOutputs()) {
+ builder.addOutputSpecs(convertOutputSpecToProto(outputSpec));
+ }
+ }
+
+ if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) {
+ for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) {
+ builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec));
+
+ }
+ }
+ return builder.build();
+ }
+
+
+ public static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) {
+ InputDescriptor inputDescriptor = null;
+ if (inputSpecProto.hasIoDescriptor()) {
+ inputDescriptor =
+ DagTypeConverters.convertInputDescriptorFromDAGPlan(inputSpecProto.getIoDescriptor());
+ }
+ InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor,
+ inputSpecProto.getPhysicalEdgeCount());
+ return inputSpec;
+ }
+
+ public static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) {
+ IOSpecProto.Builder builder = IOSpecProto.newBuilder();
+ if (inputSpec.getSourceVertexName() != null) {
+ builder.setConnectedVertexName(inputSpec.getSourceVertexName());
+ }
+ if (inputSpec.getInputDescriptor() != null) {
+ builder.setIoDescriptor(DagTypeConverters.convertToDAGPlan(inputSpec.getInputDescriptor()));
+ }
+ builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount());
+ return builder.build();
+ }
+
+ public static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) {
+ OutputDescriptor outputDescriptor = null;
+ if (outputSpecProto.hasIoDescriptor()) {
+ outputDescriptor =
+ DagTypeConverters.convertOutputDescriptorFromDAGPlan(outputSpecProto.getIoDescriptor());
+ }
+ OutputSpec outputSpec =
+ new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor,
+ outputSpecProto.getPhysicalEdgeCount());
+ return outputSpec;
+ }
+
+ public static IOSpecProto convertOutputSpecToProto(OutputSpec outputSpec) {
+ IOSpecProto.Builder builder = IOSpecProto.newBuilder();
+ if (outputSpec.getDestinationVertexName() != null) {
+ builder.setConnectedVertexName(outputSpec.getDestinationVertexName());
+ }
+ if (outputSpec.getOutputDescriptor() != null) {
+ builder.setIoDescriptor(DagTypeConverters.convertToDAGPlan(outputSpec.getOutputDescriptor()));
+ }
+ builder.setPhysicalEdgeCount(outputSpec.getPhysicalEdgeCount());
+ return builder.build();
+ }
+
+ public static GroupInputSpec getGroupInputSpecFromProto(GroupInputSpecProto groupInputSpecProto) {
+ GroupInputSpec groupSpec = new GroupInputSpec(groupInputSpecProto.getGroupName(),
+ groupInputSpecProto.getGroupVerticesList(), DagTypeConverters
+ .convertInputDescriptorFromDAGPlan(groupInputSpecProto.getMergedInputDescriptor()));
+ return groupSpec;
+ }
+
+ public static GroupInputSpecProto convertGroupInputSpecToProto(GroupInputSpec groupInputSpec) {
+ GroupInputSpecProto.Builder builder = GroupInputSpecProto.newBuilder();
+ builder.setGroupName(groupInputSpec.getGroupName());
+ builder.addAllGroupVertices(groupInputSpec.getGroupVertices());
+ builder.setMergedInputDescriptor(
+ DagTypeConverters.convertToDAGPlan(groupInputSpec.getMergedInputDescriptor()));
+ return builder.build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/proto/TezDaemonProtocol.proto
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/proto/TezDaemonProtocol.proto b/tez-ext-service-tests/src/test/proto/TezDaemonProtocol.proto
new file mode 100644
index 0000000..2f8b2e6
--- /dev/null
+++ b/tez-ext-service-tests/src/test/proto/TezDaemonProtocol.proto
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tez.test.service.rpc";
+option java_outer_classname = "TezTestServiceProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+import "DAGApiRecords.proto";
+
+message IOSpecProto {
+ optional string connected_vertex_name = 1;
+ optional TezEntityDescriptorProto io_descriptor = 2;
+ optional int32 physical_edge_count = 3;
+}
+
+message GroupInputSpecProto {
+ optional string group_name = 1;
+ repeated string group_vertices = 2;
+ optional TezEntityDescriptorProto merged_input_descriptor = 3;
+}
+
+message TaskSpecProto {
+ optional string task_attempt_id_string = 1;
+ optional string dag_name = 2;
+ optional string vertex_name = 3;
+ optional TezEntityDescriptorProto processor_descriptor = 4;
+ repeated IOSpecProto input_specs = 5;
+ repeated IOSpecProto output_specs = 6;
+ repeated GroupInputSpecProto grouped_input_specs = 7;
+ optional int32 vertex_parallelism = 8;
+}
+
+
+message SubmitWorkRequestProto {
+ optional string container_id_string = 1;
+ optional string am_host = 2;
+ optional int32 am_port = 3;
+ optional string token_identifier = 4;
+ optional bytes credentials_binary = 5;
+ optional string user = 6;
+ optional string application_id_string = 7;
+ optional int32 app_attempt_number = 8;
+ optional TaskSpecProto task_spec = 9;
+}
+
+message SubmitWorkResponseProto {
+}
+
+
+
+message RunContainerRequestProto {
+ optional string container_id_string = 1;
+ optional string am_host = 2;
+ optional int32 am_port = 3;
+ optional string token_identifier = 4;
+ optional bytes credentials_binary = 5;
+ optional string user = 6;
+ optional string application_id_string = 7;
+ optional int32 app_attempt_number = 8;
+}
+
+message RunContainerResponseProto {
+}
+
+service TezTestServiceProtocol {
+ rpc runContainer(RunContainerRequestProto) returns (RunContainerResponseProto);
+ rpc submitWork(SubmitWorkRequestProto) returns (SubmitWorkResponseProto);
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/resources/log4j.properties b/tez-ext-service-tests/src/test/resources/log4j.properties
new file mode 100644
index 0000000..531b68b
--- /dev/null
+++ b/tez-ext-service-tests/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index fd55992..3cba3ce 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -389,7 +389,7 @@ public class TezChild {
private final Throwable throwable;
private final String errorMessage;
- ContainerExecutionResult(ExitStatus exitStatus, @Nullable Throwable throwable,
+ public ContainerExecutionResult(ExitStatus exitStatus, @Nullable Throwable throwable,
@Nullable String errorMessage) {
this.exitStatus = exitStatus;
this.throwable = throwable;
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
index 766b5c3..84ebffe 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
@@ -67,7 +67,7 @@ public class TezTaskRunner implements TezUmbilical, ErrorReporter {
private final AtomicBoolean taskRunning;
private final AtomicBoolean shutdownRequested = new AtomicBoolean(false);
- TezTaskRunner(Configuration tezConf, UserGroupInformation ugi, String[] localDirs,
+ public TezTaskRunner(Configuration tezConf, UserGroupInformation ugi, String[] localDirs,
TaskSpec taskSpec, int appAttemptNumber,
Map<String, ByteBuffer> serviceConsumerMetadata, Map<String, String> serviceProviderEnvMap,
Multimap<String, String> startedInputsMap, TaskReporter taskReporter,
[11/50] [abbrv] tez git commit: TEZ-714. OutputCommitters should not
run in the main AM dispatcher thread (zjffdu)
Posted by ss...@apache.org.
TEZ-714. OutputCommitters should not run in the main AM dispatcher thread (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/d932579b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/d932579b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/d932579b
Branch: refs/heads/TEZ-2003
Commit: d932579b002f14b81836eeed75f4bf92d4ed7fbf
Parents: 5de0fb1
Author: Jeff Zhang <zj...@apache.org>
Authored: Thu Apr 16 06:39:34 2015 +0200
Committer: Jeff Zhang <zj...@apache.org>
Committed: Thu Apr 16 06:39:34 2015 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../dag/api/client/DAGClientTimelineImpl.java | 2 +
.../apache/tez/dag/api/client/DAGStatus.java | 1 +
.../apache/tez/dag/api/client/VertexStatus.java | 4 +-
tez-api/src/main/proto/DAGApiRecords.proto | 2 +
.../tez/dag/api/client/DAGStatusBuilder.java | 2 +
.../tez/dag/api/client/VertexStatusBuilder.java | 2 +
.../org/apache/tez/dag/app/dag/DAGState.java | 3 +-
.../tez/dag/app/dag/DAGTerminationCause.java | 2 +
.../tez/dag/app/dag/TaskTerminationCause.java | 2 +
.../org/apache/tez/dag/app/dag/VertexState.java | 1 +
.../tez/dag/app/dag/VertexTerminationCause.java | 8 +-
.../app/dag/event/DAGEventCommitCompleted.java | 48 +
.../tez/dag/app/dag/event/DAGEventType.java | 4 +-
.../dag/event/VertexEventCommitCompleted.java | 47 +
.../tez/dag/app/dag/event/VertexEventType.java | 5 +-
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 757 +++++---
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 559 ++++--
.../tez/dag/app/TestMockDAGAppMaster.java | 236 ++-
.../apache/tez/dag/app/dag/impl/TestCommit.java | 1682 ++++++++++++++++++
.../tez/dag/app/dag/impl/TestDAGImpl.java | 5 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 6 +-
.../tez/mapreduce/examples/ExampleDriver.java | 2 +
.../examples/MultipleCommitsExample.java | 270 +++
.../java/org/apache/tez/test/TestTezJobs.java | 78 +
25 files changed, 3294 insertions(+), 435 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4cd4557..11c7322 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-714. OutputCommitters should not run in the main AM dispatcher thread
TEZ-2323. Fix TestOrderedWordcount to use MR memory configs.
TEZ-1482. Fix memory issues for Local Mode running concurrent tasks
TEZ-2033. Update TestOrderedWordCount to add processor configs as history text
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
index 4a5a4e2..cc000df 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
@@ -479,6 +479,7 @@ public class DAGClientTimelineImpl extends DAGClient {
put("KILLED", DAGStatusStateProto.DAG_KILLED);
put("ERROR", DAGStatusStateProto.DAG_ERROR);
put("TERMINATING", DAGStatusStateProto.DAG_TERMINATING);
+ put("COMMITTING", DAGStatusStateProto.DAG_COMMITTING);
}});
private static final Map<String, VertexStatusStateProto> vertexStateProtoMap =
@@ -493,6 +494,7 @@ public class DAGClientTimelineImpl extends DAGClient {
put("KILLED", VertexStatusStateProto.VERTEX_KILLED);
put("ERROR", VertexStatusStateProto.VERTEX_ERROR);
put("TERMINATING", VertexStatusStateProto.VERTEX_TERMINATING);
+ put("COMMITTING", VertexStatusStateProto.VERTEX_COMMITTING);
}});
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
index 7e48334..f530858 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
@@ -74,6 +74,7 @@ public class DAGStatus {
case DAG_INITING:
case DAG_TERMINATING:
case DAG_RUNNING:
+ case DAG_COMMITTING:
return DAGStatus.State.RUNNING;
case DAG_SUCCEEDED:
return DAGStatus.State.SUCCEEDED;
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
index e22d6c0..61f093d 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
@@ -19,7 +19,6 @@
package org.apache.tez.dag.api.client;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.annotations.VisibleForTesting;
@@ -43,6 +42,7 @@ public class VertexStatus {
INITIALIZING,
INITED,
RUNNING,
+ COMMITTING,
SUCCEEDED,
FAILED,
KILLED,
@@ -74,6 +74,8 @@ public class VertexStatus {
return VertexStatus.State.INITED;
case VERTEX_RUNNING:
return VertexStatus.State.RUNNING;
+ case VERTEX_COMMITTING:
+ return VertexStatus.State.COMMITTING;
case VERTEX_SUCCEEDED:
return VertexStatus.State.SUCCEEDED;
case VERTEX_FAILED:
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-api/src/main/proto/DAGApiRecords.proto
----------------------------------------------------------------------
diff --git a/tez-api/src/main/proto/DAGApiRecords.proto b/tez-api/src/main/proto/DAGApiRecords.proto
index 9ac08b2..959d4e6 100644
--- a/tez-api/src/main/proto/DAGApiRecords.proto
+++ b/tez-api/src/main/proto/DAGApiRecords.proto
@@ -199,6 +199,7 @@ enum VertexStatusStateProto {
VERTEX_KILLED = 7;
VERTEX_ERROR = 8;
VERTEX_TERMINATING = 9;
+ VERTEX_COMMITTING = 10;
}
message VertexStatusProto {
@@ -217,6 +218,7 @@ enum DAGStatusStateProto {
DAG_FAILED = 5;
DAG_ERROR = 6;
DAG_TERMINATING = 7;
+ DAG_COMMITTING = 8;
}
message StringProgressPairProto {
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
index b0a2c63..0002d8b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
@@ -72,6 +72,8 @@ public class DAGStatusBuilder extends DAGStatus {
return DAGStatusStateProto.DAG_INITING;
case RUNNING:
return DAGStatusStateProto.DAG_RUNNING;
+ case COMMITTING:
+ return DAGStatusStateProto.DAG_COMMITTING;
case SUCCEEDED:
return DAGStatusStateProto.DAG_SUCCEEDED;
case FAILED:
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
index 1ea4e3d..ada3490 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
@@ -71,6 +71,8 @@ public class VertexStatusBuilder extends VertexStatus {
return VertexStatusStateProto.VERTEX_INITED;
case RUNNING:
return VertexStatusStateProto.VERTEX_RUNNING;
+ case COMMITTING:
+ return VertexStatusStateProto.VERTEX_COMMITTING;
case SUCCEEDED:
return VertexStatusStateProto.VERTEX_SUCCEEDED;
case FAILED:
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java
index b83e9a5..1acc08d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java
@@ -25,5 +25,6 @@ public enum DAGState {
FAILED,
KILLED,
ERROR,
- TERMINATING
+ TERMINATING,
+ COMMITTING,
}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java
index 5ae96a1..d8ba95d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java
@@ -42,6 +42,8 @@ public enum DAGTerminationCause {
/** In some cases, vertex could not rerun, e.g. its output been committed as a shared output of vertex group */
VERTEX_RERUN_AFTER_COMMIT,
+ VERTEX_RERUN_IN_COMMITTING,
+
/** DAG failed while trying to write recovery events */
RECOVERY_FAILURE,
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java
index eb0a2af..e2ae8da 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java
@@ -39,4 +39,6 @@ public enum TaskTerminationCause {
* throw Exception
*/
AM_USERCODE_FAILURE,
+
+ TASK_RESCHEDULE_IN_COMMITTING,
}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexState.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexState.java
index 7130c7a..1b7ac0f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexState.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexState.java
@@ -28,4 +28,5 @@ public enum VertexState {
ERROR,
TERMINATING,
RECOVERING,
+ COMMITTING,
}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java
index 2eeae3c..ebece97 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java
@@ -46,11 +46,17 @@ public enum VertexTerminationCause {
/** In some cases, vertex could not rerun, e.g. its output been committed as a shared output of vertex group */
VERTEX_RERUN_AFTER_COMMIT,
+ /** Rerun vertex while it is in committing, it would cause conflict. */
+ VERTEX_RERUN_IN_COMMITTING,
+
/** This vertex failed as it had invalid number tasks. */
INVALID_NUM_OF_TASKS,
/** This vertex failed during init. */
INIT_FAILURE,
- INTERNAL_ERROR
+ INTERNAL_ERROR,
+
+ /** error when writing recovery log */
+ RECOVERY_ERROR,
}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCommitCompleted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCommitCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCommitCompleted.java
new file mode 100644
index 0000000..2cfd60b
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCommitCompleted.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.dag.app.dag.event;
+
+import org.apache.tez.dag.app.dag.impl.DAGImpl.OutputKey;
+import org.apache.tez.dag.records.TezDAGID;
+
+public class DAGEventCommitCompleted extends DAGEvent{
+
+ private OutputKey outputKey;
+ private boolean isSucceeded;
+ private Throwable failException;
+
+ public DAGEventCommitCompleted(TezDAGID dagId, OutputKey outputKey,
+ boolean isSucceeded, Throwable failException) {
+ super(dagId, DAGEventType.DAG_COMMIT_COMPLETED);
+ this.outputKey = outputKey;
+ this.isSucceeded = isSucceeded;
+ this.failException = failException;
+ }
+
+ public OutputKey getOutputKey() {
+ return outputKey;
+ }
+
+ public boolean isSucceeded() {
+ return this.isSucceeded;
+ }
+
+ public Throwable getException() {
+ return failException;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
index 146ff4d..ea6a3cc 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
@@ -44,5 +44,7 @@ public enum DAGEventType {
// Event to trigger recovery
// Producer:AM
- DAG_RECOVER
+ DAG_RECOVER,
+
+ DAG_COMMIT_COMPLETED,
}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventCommitCompleted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventCommitCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventCommitCompleted.java
new file mode 100644
index 0000000..77272f2
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventCommitCompleted.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.dag.app.dag.event;
+
+import org.apache.tez.dag.records.TezVertexID;
+
+public class VertexEventCommitCompleted extends VertexEvent {
+
+ private String outputName;
+ private boolean isSucceeded;
+ private Throwable failException;
+
+ public VertexEventCommitCompleted(TezVertexID vertexId, String outputName,
+ boolean isSucceeded, Throwable failException) {
+ super(vertexId, VertexEventType.V_COMMIT_COMPLETED);
+ this.outputName = outputName;
+ this.isSucceeded = isSucceeded;
+ this.failException = failException;
+ }
+
+ public String getOutputName() {
+ return outputName;
+ }
+
+ public boolean isSucceeded() {
+ return isSucceeded;
+ }
+
+ public Throwable getException() {
+ return failException;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java
index aa202a4..6ea945b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java
@@ -62,6 +62,9 @@ public enum VertexEventType {
V_SOURCE_VERTEX_RECOVERED,
// Producer: Edge
- V_NULL_EDGE_INITIALIZED
+ V_NULL_EDGE_INITIALIZED,
+
+ // Committer
+ V_COMMIT_COMPLETED,
}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 3b282d6..1c93dc6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -35,6 +35,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
@@ -93,8 +94,10 @@ import org.apache.tez.dag.app.dag.StateChangeNotifier;
import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.VertexState;
import org.apache.tez.dag.app.dag.VertexTerminationCause;
+import org.apache.tez.dag.app.dag.event.CallableEvent;
import org.apache.tez.dag.app.dag.event.DAGAppMasterEventDAGFinished;
import org.apache.tez.dag.app.dag.event.DAGEvent;
+import org.apache.tez.dag.app.dag.event.DAGEventCommitCompleted;
import org.apache.tez.dag.app.dag.event.DAGEventCounterUpdate;
import org.apache.tez.dag.app.dag.event.DAGEventDiagnosticsUpdate;
import org.apache.tez.dag.app.dag.event.DAGEventRecoverEvent;
@@ -127,9 +130,13 @@ import org.apache.tez.dag.utils.TezBuilderUtils;
import org.apache.tez.runtime.api.OutputCommitter;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
/** Implementation of Job interface. Maintains the state machines of Job.
* The read and write calls use ReadWriteLock for concurrency.
@@ -158,8 +165,9 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private final TaskHeartbeatHandler taskHeartbeatHandler;
private final Object tasksSyncHandle = new Object();
- private volatile boolean committedOrAborted = false;
- private volatile boolean allOutputsCommitted = false;
+ private AtomicBoolean committed = new AtomicBoolean(false);
+ private AtomicBoolean aborted = new AtomicBoolean(false);
+ private AtomicBoolean commitCanceled = new AtomicBoolean(false);
boolean commitAllOutputsOnSuccess = true;
@VisibleForTesting
@@ -201,6 +209,11 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private static final DagStateChangedCallback STATE_CHANGED_CALLBACK = new DagStateChangedCallback();
+ @VisibleForTesting
+ Map<OutputKey, ListenableFuture<Void>> commitFutures
+ = new HashMap<OutputKey, ListenableFuture<Void>>();
+ private Set<OutputKey> succeededCommits = new HashSet<OutputKey>();
+
private static final DiagnosticsUpdateTransition
DIAGNOSTIC_UPDATE_TRANSITION = new DiagnosticsUpdateTransition();
private static final InternalErrorTransition
@@ -209,6 +222,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
new CounterUpdateTransition();
private static final DAGSchedulerUpdateTransition
DAG_SCHEDULER_UPDATE_TRANSITION = new DAGSchedulerUpdateTransition();
+ private static final CommitCompletedTransition COMMIT_COMPLETED_TRANSITION =
+ new CommitCompletedTransition();
protected static final
StateMachineFactory<DAGImpl, DAGState, DAGEventType, DAGEvent>
@@ -259,7 +274,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
// Transitions from RUNNING state
.addTransition
(DAGState.RUNNING,
- EnumSet.of(DAGState.RUNNING, DAGState.SUCCEEDED, DAGState.TERMINATING,DAGState.FAILED),
+ EnumSet.of(DAGState.RUNNING, DAGState.COMMITTING,
+ DAGState.SUCCEEDED, DAGState.TERMINATING,DAGState.FAILED),
DAGEventType.DAG_VERTEX_COMPLETED,
new VertexCompletedTransition())
.addTransition(DAGState.RUNNING, EnumSet.of(DAGState.RUNNING, DAGState.TERMINATING),
@@ -279,6 +295,35 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
DAGState.RUNNING,
DAGState.ERROR, DAGEventType.INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
+ .addTransition(DAGState.RUNNING,
+ EnumSet.of(DAGState.RUNNING, DAGState.TERMINATING),
+ DAGEventType.DAG_COMMIT_COMPLETED,
+ new CommitCompletedWhileRunning())
+
+ // Transitions from COMMITTING state.
+ .addTransition(DAGState.COMMITTING,
+ EnumSet.of(DAGState.COMMITTING, DAGState.TERMINATING, DAGState.FAILED, DAGState.SUCCEEDED),
+ DAGEventType.DAG_COMMIT_COMPLETED,
+ COMMIT_COMPLETED_TRANSITION)
+ .addTransition(DAGState.COMMITTING, DAGState.TERMINATING,
+ DAGEventType.DAG_KILL,
+ new DAGKilledWhileCommittingTransition())
+ .addTransition(
+ DAGState.COMMITTING,
+ DAGState.ERROR,
+ DAGEventType.INTERNAL_ERROR,
+ INTERNAL_ERROR_TRANSITION)
+ .addTransition(DAGState.COMMITTING, DAGState.COMMITTING,
+ DAGEventType.DAG_DIAGNOSTIC_UPDATE,
+ DIAGNOSTIC_UPDATE_TRANSITION)
+ .addTransition(DAGState.COMMITTING, DAGState.COMMITTING,
+ DAGEventType.DAG_SCHEDULER_UPDATE,
+ DAG_SCHEDULER_UPDATE_TRANSITION)
+ .addTransition(DAGState.COMMITTING, DAGState.TERMINATING,
+ DAGEventType.DAG_VERTEX_RERUNNING,
+ new VertexRerunWhileCommitting())
+ .addTransition(DAGState.COMMITTING, DAGState.COMMITTING,
+ DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
// Transitions from TERMINATING state.
.addTransition
@@ -296,6 +341,11 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
DAGState.TERMINATING,
DAGState.ERROR, DAGEventType.INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
+ .addTransition(
+ DAGState.TERMINATING,
+ EnumSet.of(DAGState.TERMINATING, DAGState.FAILED, DAGState.KILLED, DAGState.ERROR),
+ DAGEventType.DAG_COMMIT_COMPLETED,
+ COMMIT_COMPLETED_TRANSITION)
// Ignore-able events
.addTransition(DAGState.TERMINATING, DAGState.TERMINATING,
@@ -885,144 +935,125 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
}
- private boolean commitOutput(String outputName, OutputCommitter outputCommitter) {
+ private void commitOutput(OutputCommitter outputCommitter) throws Exception {
final OutputCommitter committer = outputCommitter;
- try {
- getDagUGI().doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- committer.commitOutput();
- return null;
- }
- });
- return true;
- } catch (Exception e) {
- LOG.info("Exception in committing output: " + outputName, e);
- }
- return false;
+ getDagUGI().doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ committer.commitOutput();
+ return null;
+ }
+ });
}
- private synchronized boolean commitOrAbortOutputs(boolean dagSucceeded) {
- if (this.committedOrAborted) {
- LOG.info("Ignoring multiple output commit/abort");
- return this.allOutputsCommitted;
+ // either commit when all vertices are completed or just finish if there's no committer
+ private synchronized DAGState commitOrFinish() {
+
+ // commit all other outputs
+ // we come here for successful dag completion and when outputs need to be
+ // committed at the end for all or none visibility
+ Map<OutputKey, CallableEvent> commitEvents = new HashMap<OutputKey, CallableEvent>();
+ // commit all shared outputs
+ for (final VertexGroupInfo groupInfo : vertexGroups.values()) {
+ if (!groupInfo.outputs.isEmpty()) {
+ groupInfo.committed = true;
+ final Vertex v = getVertex(groupInfo.groupMembers.iterator().next());
+ for (final String outputName : groupInfo.outputs) {
+ final OutputKey outputKey = new OutputKey(outputName, groupInfo.groupName, true);
+ CommitCallback groupCommitCallback = new CommitCallback(outputKey);
+ CallableEvent groupCommitCallableEvent = new CallableEvent(groupCommitCallback) {
+ @Override
+ public Void call() throws Exception {
+ OutputCommitter committer = v.getOutputCommitters().get(outputName);
+ LOG.info("Committing output: " + outputKey);
+ commitOutput(committer);
+ return null;
+ }
+ };
+ commitEvents.put(outputKey, groupCommitCallableEvent);
+ }
+ }
}
- LOG.info("Calling DAG commit/abort for dag: " + getID());
- this.committedOrAborted = true;
- boolean successfulOutputsAlreadyCommitted = !commitAllOutputsOnSuccess;
- boolean failedWhileCommitting = false;
- if (dagSucceeded && !successfulOutputsAlreadyCommitted) {
- // commit all shared outputs
+ for (final Vertex vertex : vertices.values()) {
+ if (vertex.getOutputCommitters() == null) {
+ LOG.info("No output committers for vertex: " + vertex.getLogIdentifier());
+ continue;
+ }
+ Map<String, OutputCommitter> outputCommitters =
+ new HashMap<String, OutputCommitter>(vertex.getOutputCommitters());
+ Set<String> sharedOutputs = vertex.getSharedOutputs();
+ // remove shared outputs
+ if (sharedOutputs != null) {
+ Iterator<Map.Entry<String, OutputCommitter>> iter = outputCommitters
+ .entrySet().iterator();
+ while (iter.hasNext()) {
+ if (sharedOutputs.contains(iter.next().getKey())) {
+ iter.remove();
+ }
+ }
+ }
+ if (outputCommitters.isEmpty()) {
+ LOG.info("No exclusive output committers for vertex: " + vertex.getLogIdentifier());
+ continue;
+ }
+ for (final Map.Entry<String, OutputCommitter> entry : outputCommitters.entrySet()) {
+ if (vertex.getState() != VertexState.SUCCEEDED) {
+ throw new TezUncheckedException("Vertex: " + vertex.getLogIdentifier() +
+ " not in SUCCEEDED state. State= " + vertex.getState());
+ }
+ OutputKey outputKey = new OutputKey(entry.getKey(), vertex.getName(), false);
+ CommitCallback commitCallback = new CommitCallback(outputKey);
+ CallableEvent commitCallableEvent = new CallableEvent(commitCallback) {
+ @Override
+ public Void call() throws Exception {
+ LOG.info("Committing output: " + entry.getKey() + " for vertex: "
+ + vertex.getLogIdentifier() + ", outputName: " + entry.getKey());
+ commitOutput(entry.getValue());
+ return null;
+ }
+ };
+ commitEvents.put(outputKey, commitCallableEvent);
+ }
+ }
+
+ if (!commitEvents.isEmpty()) {
try {
+ LOG.info("Start writing dag commit event, " + getID());
appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(),
new DAGCommitStartedEvent(getID(), clock.getTime())));
} catch (IOException e) {
LOG.error("Failed to send commit event to history/recovery handler", e);
trySetTerminationCause(DAGTerminationCause.RECOVERY_FAILURE);
- return false;
+ return finished(DAGState.FAILED);
}
- for (VertexGroupInfo groupInfo : vertexGroups.values()) {
- if (failedWhileCommitting) {
- break;
- }
- if (!groupInfo.outputs.isEmpty()) {
- groupInfo.committed = true;
- Vertex v = getVertex(groupInfo.groupMembers.iterator().next());
- for (String outputName : groupInfo.outputs) {
- OutputCommitter committer = v.getOutputCommitters().get(outputName);
- LOG.info("Committing output: " + outputName + " for group: " + groupInfo.groupName);
- if (!commitOutput(outputName, committer)) {
- failedWhileCommitting = true;
- break;
- }
- }
- }
- }
- // commit all other outputs
- // we come here for successful dag completion and when outputs need to be
- // committed at the end for all or none visibility
- for (Vertex vertex : vertices.values()) {
- if (failedWhileCommitting) {
- break;
- }
- if (vertex.getOutputCommitters() == null) {
- LOG.info("No output committers for vertex: " + vertex.getLogIdentifier());
- continue;
- }
- Map<String, OutputCommitter> outputCommitters =
- new HashMap<String, OutputCommitter>(vertex.getOutputCommitters());
- Set<String> sharedOutputs = vertex.getSharedOutputs();
- // remove shared outputs
- if (sharedOutputs != null) {
- Iterator<Map.Entry<String, OutputCommitter>> iter = outputCommitters
- .entrySet().iterator();
- while (iter.hasNext()) {
- if (sharedOutputs.contains(iter.next().getKey())) {
- iter.remove();
- }
- }
- }
- if (outputCommitters.isEmpty()) {
- LOG.info("No exclusive output committers for vertex: " + vertex.getLogIdentifier());
- continue;
- }
- for (Map.Entry<String, OutputCommitter> entry : outputCommitters.entrySet()) {
- LOG.info("Committing output: " + entry.getKey() + " for vertex: "
- + vertex.getLogIdentifier());
- if (vertex.getState() != VertexState.SUCCEEDED) {
- throw new TezUncheckedException("Vertex: " + vertex.getLogIdentifier() +
- " not in SUCCEEDED state. State= " + vertex.getState());
- }
- if (!commitOutput(entry.getKey(), entry.getValue())) {
- failedWhileCommitting = true;
- break;
- }
- }
+ for (Map.Entry<OutputKey,CallableEvent> entry : commitEvents.entrySet()) {
+ ListenableFuture<Void> commitFuture = appContext.getExecService().submit(entry.getValue());
+ Futures.addCallback(commitFuture, entry.getValue().getCallback());
+ commitFutures.put(entry.getKey(), commitFuture);
}
}
- if (failedWhileCommitting) {
- LOG.info("DAG: " + getID() + " failed while committing");
+ if (commitFutures.isEmpty()) {
+ // no commit needs to be done
+ return finished(DAGState.SUCCEEDED);
+ } else {
+ return DAGState.COMMITTING;
}
+ }
- if (!dagSucceeded || failedWhileCommitting) {
- // come here because dag failed or
- // dag succeeded and all or none semantics were on and a commit failed
- for (Vertex vertex : vertices.values()) {
- Map<String, OutputCommitter> outputCommitters = vertex
- .getOutputCommitters();
- if (outputCommitters == null || outputCommitters.isEmpty()) {
- LOG.info("No output committers for vertex: " + vertex.getLogIdentifier());
- continue;
- }
- for (Map.Entry<String, OutputCommitter> entry : outputCommitters
- .entrySet()) {
- final OutputCommitter committer = entry.getValue();
- if (commitAllOutputsOnSuccess // commit all outputs on success
- || vertex.getState() != VertexState.SUCCEEDED // never commit unsuccessful outputs
- ) {
- LOG.info("Aborting output: " + entry.getKey() + " for vertex: "
- + vertex.getLogIdentifier());
- try {
- getDagUGI().doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- committer.abortOutput(VertexStatus.State.FAILED);
- return null;
- }
- });
- } catch (Exception e) {
- LOG.info("Exception in aborting output: " + entry.getKey()
- + " for vertex: " + vertex.getLogIdentifier(), e);
- }
- }
- // else successful outputs have already been committed
- }
- }
+ private void abortOutputs() {
+ if (this.aborted.getAndSet(true)) {
+ LOG.info("Ignoring multiple output abort");
+ return ;
+ }
+ // come here because dag failed or
+ // dag succeeded and all or none semantics were on and a commit failed.
+ // Some output may be aborted multiple times if it is shared output.
+ // It should be OK for it to be aborted multiple times.
+ for (Vertex vertex : vertices.values()) {
+ ((VertexImpl)vertex).abortVertex(VertexStatus.State.FAILED);
}
- allOutputsCommitted = !failedWhileCommitting;
- return allOutputsCommitted;
}
@Override
@@ -1124,16 +1155,16 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
new DAGHistoryEvent(dagId, finishEvt));
}
- static DAGState checkDAGForCompletion(DAGImpl dag) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Checking dag completion"
- + ", numCompletedVertices=" + dag.numCompletedVertices
- + ", numSuccessfulVertices=" + dag.numSuccessfulVertices
- + ", numFailedVertices=" + dag.numFailedVertices
- + ", numKilledVertices=" + dag.numKilledVertices
- + ", numVertices=" + dag.numVertices
- + ", terminationCause=" + dag.terminationCause);
- }
+ // triggered by vertex_complete
+ static DAGState checkVerticesForCompletion(DAGImpl dag) {
+ LOG.info("Checking vertices for DAG completion"
+ + ", numCompletedVertices=" + dag.numCompletedVertices
+ + ", numSuccessfulVertices=" + dag.numSuccessfulVertices
+ + ", numFailedVertices=" + dag.numFailedVertices
+ + ", numKilledVertices=" + dag.numKilledVertices
+ + ", numVertices=" + dag.numVertices
+ + ", commitInProgress=" + dag.commitFutures.size()
+ + ", terminationCause=" + dag.terminationCause);
// log in case of accounting error.
if (dag.numCompletedVertices > dag.numVertices) {
@@ -1144,67 +1175,124 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
if (dag.numCompletedVertices == dag.numVertices) {
- dag.setFinishTime();
//Only succeed if vertices complete successfully and no terminationCause is registered.
if(dag.numSuccessfulVertices == dag.numVertices && dag.terminationCause == null) {
- return dag.finished(DAGState.SUCCEEDED);
- }
- if(dag.terminationCause == DAGTerminationCause.DAG_KILL ){
- String diagnosticMsg = "DAG killed due to user-initiated kill." +
- " failedVertices:" + dag.numFailedVertices +
- " killedVertices:" + dag.numKilledVertices;
- LOG.info(diagnosticMsg);
- dag.addDiagnostic(diagnosticMsg);
- return dag.finished(DAGState.KILLED);
- }
- if(dag.terminationCause == DAGTerminationCause.VERTEX_FAILURE ){
- String diagnosticMsg = "DAG failed due to vertex failure." +
- " failedVertices:" + dag.numFailedVertices +
- " killedVertices:" + dag.numKilledVertices;
- LOG.info(diagnosticMsg);
- dag.addDiagnostic(diagnosticMsg);
- return dag.finished(DAGState.FAILED);
- }
- if(dag.terminationCause == DAGTerminationCause.COMMIT_FAILURE ){
- String diagnosticMsg = "DAG failed due to commit failure." +
- " failedVertices:" + dag.numFailedVertices +
- " killedVertices:" + dag.numKilledVertices;
- LOG.info(diagnosticMsg);
- dag.addDiagnostic(diagnosticMsg);
- return dag.finished(DAGState.FAILED);
+ if (dag.commitAllOutputsOnSuccess && !dag.committed.getAndSet(true)) {
+ // start dag commit if there's any commit or just finish dag if no commit
+ return dag.commitOrFinish();
+ } else if (!dag.commitFutures.isEmpty()) {
+ // vertex group commits are running
+ return DAGState.COMMITTING;
+ } else {
+ // no vertex group commits or vertex group commits are done
+ return dag.finished(DAGState.SUCCEEDED);
+ }
+ } else {
+ if (dag.commitFutures.isEmpty()) {
+ return finishWithTerminationCause(dag);
+ } else {
+ return DAGState.TERMINATING;
+ }
}
- if(dag.terminationCause == DAGTerminationCause.VERTEX_RERUN_AFTER_COMMIT ){
- String diagnosticMsg = "DAG failed due to vertex rerun after commit." +
- " failedVertices:" + dag.numFailedVertices +
- " killedVertices:" + dag.numKilledVertices;
- LOG.info(diagnosticMsg);
- dag.addDiagnostic(diagnosticMsg);
- return dag.finished(DAGState.FAILED);
+ }
+
+ //return the current state, Job not finished yet
+ return dag.getInternalState();
+ }
+
+ // triggered by commit_complete
+ static DAGState checkCommitsForCompletion(DAGImpl dag) {
+ LOG.info("Checking commits for DAG completion"
+ + ", numCompletedVertices=" + dag.numCompletedVertices
+ + ", numSuccessfulVertices=" + dag.numSuccessfulVertices
+ + ", numFailedVertices=" + dag.numFailedVertices
+ + ", numKilledVertices=" + dag.numKilledVertices
+ + ", numVertices=" + dag.numVertices
+ + ", commitInProgress=" + dag.commitFutures.size()
+ + ", terminationCause=" + dag.terminationCause);
+
+ // terminationCause is null means commit is succeeded, otherwise terminationCause will be set.
+ if (dag.terminationCause == null) {
+ Preconditions.checkState(dag.getState() == DAGState.COMMITTING,
+ "DAG should be in COMMITTING state, but in " + dag.getState());
+ if (!dag.commitFutures.isEmpty()) {
+ // pending commits are running
+ return DAGState.COMMITTING;
+ } else {
+ return dag.finished(DAGState.SUCCEEDED);
}
- if(dag.terminationCause == DAGTerminationCause.RECOVERY_FAILURE ){
- String diagnosticMsg = "DAG failed due to failure in recovery handling." +
- " failedVertices:" + dag.numFailedVertices +
- " killedVertices:" + dag.numKilledVertices;
- LOG.info(diagnosticMsg);
- dag.addDiagnostic(diagnosticMsg);
- return dag.finished(DAGState.FAILED);
+ } else {
+ if (!dag.commitFutures.isEmpty()) {
+ // pending commits are running
+ return DAGState.TERMINATING;
+ } else {
+ return finishWithTerminationCause(dag);
}
+ }
+ }
- // catch all
- String diagnosticMsg = "All vertices complete, but cannot determine final state of DAG"
- + ", numCompletedVertices=" + dag.numCompletedVertices
- + ", numSuccessfulVertices=" + dag.numSuccessfulVertices
- + ", numFailedVertices=" + dag.numFailedVertices
- + ", numKilledVertices=" + dag.numKilledVertices
- + ", numVertices=" + dag.numVertices
- + ", terminationCause=" + dag.terminationCause;
- LOG.error(diagnosticMsg);
+ private static DAGState finishWithTerminationCause(DAGImpl dag) {
+ if(dag.terminationCause == DAGTerminationCause.DAG_KILL ){
+ String diagnosticMsg = "DAG killed due to user-initiated kill." +
+ " failedVertices:" + dag.numFailedVertices +
+ " killedVertices:" + dag.numKilledVertices;
+ LOG.info(diagnosticMsg);
dag.addDiagnostic(diagnosticMsg);
- return dag.finished(DAGState.ERROR);
+ return dag.finished(DAGState.KILLED);
+ }
+ if(dag.terminationCause == DAGTerminationCause.VERTEX_FAILURE ){
+ String diagnosticMsg = "DAG failed due to vertex failure." +
+ " failedVertices:" + dag.numFailedVertices +
+ " killedVertices:" + dag.numKilledVertices;
+ LOG.info(diagnosticMsg);
+ dag.addDiagnostic(diagnosticMsg);
+ return dag.finished(DAGState.FAILED);
+ }
+ if(dag.terminationCause == DAGTerminationCause.COMMIT_FAILURE ){
+ String diagnosticMsg = "DAG failed due to commit failure." +
+ " failedVertices:" + dag.numFailedVertices +
+ " killedVertices:" + dag.numKilledVertices;
+ LOG.info(diagnosticMsg);
+ dag.addDiagnostic(diagnosticMsg);
+ return dag.finished(DAGState.FAILED);
+ }
+ if(dag.terminationCause == DAGTerminationCause.VERTEX_RERUN_AFTER_COMMIT ){
+ String diagnosticMsg = "DAG failed due to vertex rerun after commit." +
+ " failedVertices:" + dag.numFailedVertices +
+ " killedVertices:" + dag.numKilledVertices;
+ LOG.info(diagnosticMsg);
+ dag.addDiagnostic(diagnosticMsg);
+ return dag.finished(DAGState.FAILED);
+ }
+ if(dag.terminationCause == DAGTerminationCause.VERTEX_RERUN_IN_COMMITTING ){
+ String diagnosticMsg = "DAG failed due to vertex rerun in commit." +
+ " failedVertices:" + dag.numFailedVertices +
+ " killedVertices:" + dag.numKilledVertices;
+ LOG.info(diagnosticMsg);
+ dag.addDiagnostic(diagnosticMsg);
+ return dag.finished(DAGState.FAILED);
+ }
+ if(dag.terminationCause == DAGTerminationCause.RECOVERY_FAILURE ){
+ String diagnosticMsg = "DAG failed due to failure in recovery handling." +
+ " failedVertices:" + dag.numFailedVertices +
+ " killedVertices:" + dag.numKilledVertices;
+ LOG.info(diagnosticMsg);
+ dag.addDiagnostic(diagnosticMsg);
+ return dag.finished(DAGState.FAILED);
}
- //return the current state, Job not finished yet
- return dag.getInternalState();
+ // catch all
+ String diagnosticMsg = "All vertices complete, but cannot determine final state of DAG"
+ + ", numCompletedVertices=" + dag.numCompletedVertices
+ + ", numSuccessfulVertices=" + dag.numSuccessfulVertices
+ + ", numFailedVertices=" + dag.numFailedVertices
+ + ", numKilledVertices=" + dag.numKilledVertices
+ + ", numVertices=" + dag.numVertices
+ + ", commitInProgress=" + dag.commitFutures.size()
+ + ", terminationCause=" + dag.terminationCause;
+ LOG.error(diagnosticMsg);
+ dag.addDiagnostic(diagnosticMsg);
+ return dag.finished(DAGState.ERROR);
}
private void updateCpuCounters() {
@@ -1221,13 +1309,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
setFinishTime();
}
- boolean allOutputsCommitted = commitOrAbortOutputs(finalState == DAGState.SUCCEEDED);
-
- if (finalState == DAGState.SUCCEEDED && !allOutputsCommitted) {
- finalState = DAGState.FAILED;
- trySetTerminationCause(DAGTerminationCause.COMMIT_FAILURE);
- }
-
boolean recoveryError = false;
// update cpu time counters before finishing the dag
@@ -1246,6 +1327,10 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
recoveryError = true;
}
+ if (finalState != DAGState.SUCCEEDED) {
+ abortOutputs();
+ }
+
if (recoveryError) {
eventHandler.handle(new DAGAppMasterEventDAGFinished(getID(), DAGState.ERROR));
} else {
@@ -1741,6 +1826,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
public void transition(DAGImpl job, DAGEvent event) {
job.addDiagnostic("Job received Kill while in RUNNING state.");
job.enactKill(DAGTerminationCause.DAG_KILL, VertexTerminationCause.DAG_KILL);
+ // Commit may happen when dag is still in RUNNING (vertex group commit)
+ job.cancelCommits();
// TODO Metrics
//job.metrics.endRunningJob(job);
}
@@ -1748,6 +1835,29 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
+ private static class DAGKilledWhileCommittingTransition
+ implements SingleArcTransition<DAGImpl, DAGEvent> {
+
+ @Override
+ public void transition(DAGImpl dag, DAGEvent event) {
+ String diag = "DAG received Kill while in COMMITTING state.";
+ LOG.info(diag);
+ dag.addDiagnostic(diag);
+ dag.cancelCommits();
+ dag.trySetTerminationCause(DAGTerminationCause.DAG_KILL);
+ }
+ }
+
+ private void cancelCommits() {
+ if (!this.commitCanceled.getAndSet(true)) {
+ for (Map.Entry<OutputKey, ListenableFuture<Void>> entry : commitFutures.entrySet()) {
+ OutputKey outputKey = entry.getKey();
+ LOG.info("Canceling commit of output=" + outputKey);
+ entry.getValue().cancel(true);
+ }
+ }
+ }
+
private static class VertexCompletedTransition implements
MultipleArcTransition<DAGImpl, DAGEvent, DAGState> {
@@ -1773,11 +1883,13 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
else if (vertexEvent.getVertexState() == VertexState.FAILED) {
job.enactKill(
DAGTerminationCause.VERTEX_FAILURE, VertexTerminationCause.OTHER_VERTEX_FAILURE);
+ job.cancelCommits();
job.vertexFailed(vertex);
forceTransitionToKillWait = true;
}
else if (vertexEvent.getVertexState() == VertexState.KILLED) {
job.vertexKilled(vertex);
+ job.cancelCommits();
forceTransitionToKillWait = true;
}
@@ -1791,8 +1903,9 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
+ ", numVertices=" + job.numVertices);
// if the job has not finished but a failure/kill occurred, then force the transition to KILL_WAIT.
- DAGState state = checkDAGForCompletion(job);
+ DAGState state = checkVerticesForCompletion(job);
if(state == DAGState.RUNNING && forceTransitionToKillWait){
+ job.cancelCommits();
return DAGState.TERMINATING;
}
else {
@@ -1830,7 +1943,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private boolean vertexSucceeded(Vertex vertex) {
numSuccessfulVertices++;
- boolean failedCommit = false;
boolean recoveryFailed = false;
if (!commitAllOutputsOnSuccess) {
// committing successful outputs immediately. check for shared outputs
@@ -1855,7 +1967,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
continue;
}
groupInfo.committed = true;
- Vertex v = getVertex(groupInfo.groupMembers.iterator().next());
+ final Vertex v = getVertex(groupInfo.groupMembers.iterator().next());
try {
appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(),
new VertexGroupCommitStartedEvent(dagId, groupInfo.groupName,
@@ -1863,48 +1975,33 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
} catch (IOException e) {
LOG.error("Failed to send commit recovery event to handler", e);
recoveryFailed = true;
- failedCommit = true;
}
- if (!failedCommit) {
- for (String outputName : groupInfo.outputs) {
- OutputCommitter committer = v.getOutputCommitters().get(outputName);
- LOG.info("Committing output: " + outputName);
- if (!commitOutput(outputName, committer)) {
- // using same logic as vertex level commit. stop after first failure.
- failedCommit = true;
- break;
- }
+ if (!recoveryFailed) {
+ for (final String outputName : groupInfo.outputs) {
+ OutputKey outputKey = new OutputKey(outputName, groupInfo.groupName, true);
+ CommitCallback groupCommitCallback = new CommitCallback(outputKey);
+ CallableEvent groupCommitCallableEvent = new CallableEvent(groupCommitCallback) {
+ public Void call() throws Exception {
+ OutputCommitter committer = v.getOutputCommitters().get(outputName);
+ LOG.info("Committing output: " + outputName);
+ commitOutput(committer);
+ return null;
+ };
+ };
+ ListenableFuture<Void> groupCommitFuture = appContext.getExecService().submit(groupCommitCallableEvent);
+ Futures.addCallback(groupCommitFuture, groupCommitCallableEvent.getCallback());
+ commitFutures.put(outputKey, groupCommitFuture);
}
}
- if (failedCommit) {
- break;
- }
- try {
- appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(),
- new VertexGroupCommitFinishedEvent(dagId, groupInfo.groupName,
- clock.getTime())));
- } catch (IOException e) {
- LOG.error("Failed to send commit recovery event to handler", e);
- recoveryFailed = true;
- failedCommit = true;
- }
}
}
}
-
- if (failedCommit) {
- LOG.info("Aborting job due to failure in commit.");
- if (!recoveryFailed) {
- enactKill(DAGTerminationCause.COMMIT_FAILURE,
- VertexTerminationCause.COMMIT_FAILURE);
- } else {
- LOG.info("Recovery failure occurred during commit");
- enactKill(DAGTerminationCause.RECOVERY_FAILURE,
- VertexTerminationCause.COMMIT_FAILURE);
- }
+ if (recoveryFailed) {
+ LOG.info("Recovery failure occurred during commit");
+ enactKill(DAGTerminationCause.RECOVERY_FAILURE,
+ VertexTerminationCause.COMMIT_FAILURE);
}
-
- return !failedCommit;
+ return !recoveryFailed;
}
private boolean vertexReRunning(Vertex vertex) {
@@ -1983,29 +2080,109 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
private static class DAGSchedulerUpdateTransition implements
- SingleArcTransition<DAGImpl, DAGEvent> {
- @Override
- public void transition(DAGImpl dag, DAGEvent event) {
- DAGEventSchedulerUpdate sEvent = (DAGEventSchedulerUpdate) event;
- switch(sEvent.getUpdateType()) {
- case TA_SCHEDULE:
- dag.dagScheduler.scheduleTask(sEvent);
- break;
- case TA_SCHEDULED:
- DAGEventSchedulerUpdateTAAssigned taEvent =
- (DAGEventSchedulerUpdateTAAssigned) sEvent;
- dag.dagScheduler.taskScheduled(taEvent);
- break;
- case TA_SUCCEEDED:
- dag.dagScheduler.taskSucceeded(sEvent);
- break;
- default:
- throw new TezUncheckedException("Unknown DAGEventSchedulerUpdate:"
- + sEvent.getUpdateType());
+ SingleArcTransition<DAGImpl, DAGEvent> {
+ @Override
+ public void transition(DAGImpl dag, DAGEvent event) {
+ DAGEventSchedulerUpdate sEvent = (DAGEventSchedulerUpdate) event;
+ switch(sEvent.getUpdateType()) {
+ case TA_SCHEDULE:
+ dag.dagScheduler.scheduleTask(sEvent);
+ break;
+ case TA_SCHEDULED:
+ DAGEventSchedulerUpdateTAAssigned taEvent =
+ (DAGEventSchedulerUpdateTAAssigned) sEvent;
+ dag.dagScheduler.taskScheduled(taEvent);
+ break;
+ case TA_SUCCEEDED:
+ dag.dagScheduler.taskSucceeded(sEvent);
+ break;
+ default:
+ throw new TezUncheckedException("Unknown DAGEventSchedulerUpdate:"
+ + sEvent.getUpdateType());
+ }
}
}
-}
+ private static class CommitCompletedWhileRunning implements
+ MultipleArcTransition<DAGImpl, DAGEvent, DAGState>{
+
+ @Override
+ public DAGState transition(DAGImpl dag, DAGEvent event) {
+ DAGEventCommitCompleted commitCompletedEvent = (DAGEventCommitCompleted)event;
+ if (dag.commitCompleted(commitCompletedEvent)) {
+ return DAGState.RUNNING;
+ } else {
+ return DAGState.TERMINATING;
+ }
+ }
+
+ }
+
+ private static class CommitCompletedTransition implements
+ MultipleArcTransition<DAGImpl, DAGEvent, DAGState>{
+
+ @Override
+ public DAGState transition(DAGImpl dag, DAGEvent event) {
+ DAGEventCommitCompleted commitCompletedEvent = (DAGEventCommitCompleted)event;
+ dag.commitCompleted(commitCompletedEvent);
+ return checkCommitsForCompletion(dag);
+ }
+ }
+
+ private boolean commitCompleted(DAGEventCommitCompleted commitCompletedEvent) {
+ Preconditions.checkState(commitFutures.remove(commitCompletedEvent.getOutputKey()) != null,
+ "Unknown commit:" + commitCompletedEvent.getOutputKey());
+
+ boolean commitFailed = false;
+ boolean recoveryFailed = false;
+ if (commitCompletedEvent.isSucceeded()) {
+ LOG.info("Commit succeeded for output:" + commitCompletedEvent.getOutputKey());
+ succeededCommits.add(commitCompletedEvent.getOutputKey());
+ if (!commitAllOutputsOnSuccess) {
+ try {
+ appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(),
+ new VertexGroupCommitFinishedEvent(getID(), commitCompletedEvent.getOutputKey().getEntityName(),
+ clock.getTime())));
+ } catch (IOException e) {
+ String diag = "Failed to send commit recovery event to handler, " + ExceptionUtils.getStackTrace(e);
+ addDiagnostic(diag);
+ LOG.error(diag);
+ recoveryFailed = true;
+ }
+ }
+ } else {
+ String diag = "Commit failed for output: " + commitCompletedEvent.getOutputKey()
+ + ", " + ExceptionUtils.getStackTrace(commitCompletedEvent.getException());
+ addDiagnostic(diag);
+ LOG.error(diag);
+ commitFailed = true;
+ }
+
+ if (commitFailed) {
+ enactKill(DAGTerminationCause.COMMIT_FAILURE, VertexTerminationCause.OTHER_VERTEX_FAILURE);
+ cancelCommits();
+ }
+ if (recoveryFailed){
+ enactKill(DAGTerminationCause.RECOVERY_FAILURE, VertexTerminationCause.OTHER_VERTEX_FAILURE);
+ cancelCommits();
+ }
+ return !commitFailed && !recoveryFailed;
+ }
+
+
+ private static class VertexRerunWhileCommitting implements
+ SingleArcTransition<DAGImpl, DAGEvent> {
+
+ @Override
+ public void transition(DAGImpl dag, DAGEvent event) {
+ LOG.info("Vertex rerun while dag it is COMMITTING");
+ dag.cancelCommits();
+ dag.trySetTerminationCause(DAGTerminationCause.VERTEX_RERUN_IN_COMMITTING);
+ }
+
+ }
+
+ // TODO TEZ-2250 go to TERMINATING to wait for all vertices and commits completed
private static class InternalErrorTransition implements
SingleArcTransition<DAGImpl, DAGEvent> {
@Override
@@ -2016,6 +2193,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
job.enactKill(DAGTerminationCause.INTERNAL_ERROR,
VertexTerminationCause.INTERNAL_ERROR);
job.setFinishTime();
+ job.cancelCommits();
job.finished(DAGState.ERROR);
}
}
@@ -2036,4 +2214,83 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
readLock.unlock();
}
}
+
+ // output of either vertex or vertex group
+ public static class OutputKey {
+ String outputName;
+ String entityName; // vertex name or vertex group name
+ boolean isVertexGroupOutput;
+
+ public OutputKey(String outputName, String entityName, boolean isVertexGroupOutput) {
+ super();
+ this.outputName = outputName;
+ this.entityName = entityName;
+ this.isVertexGroupOutput = isVertexGroupOutput;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result
+ + ((entityName == null) ? 0 : entityName.hashCode());
+ result = prime * result + (isVertexGroupOutput ? 1231 : 1237);
+ result = prime * result
+ + ((outputName == null) ? 0 : outputName.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ OutputKey other = (OutputKey) obj;
+ if (entityName == null) {
+ if (other.entityName != null)
+ return false;
+ } else if (!entityName.equals(other.entityName))
+ return false;
+ if (isVertexGroupOutput != other.isVertexGroupOutput)
+ return false;
+ if (outputName == null) {
+ if (other.outputName != null)
+ return false;
+ } else if (!outputName.equals(other.outputName))
+ return false;
+ return true;
+ }
+
+ public String getEntityName() {
+ return entityName;
+ }
+
+ @Override
+ public String toString() {
+ return "outputName:" + outputName + " of vertex/vertexGroup:" + entityName
+ + " isVertexGroupOutput:" + isVertexGroupOutput;
+ }
+ }
+
+ private class CommitCallback implements FutureCallback<Void> {
+
+ private OutputKey outputKey;
+
+ public CommitCallback(OutputKey outputKey) {
+ this.outputKey = outputKey;
+ }
+
+ @Override
+ public void onSuccess(Void result) {
+ eventHandler.handle(new DAGEventCommitCompleted(dagId, outputKey, true, null));
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ eventHandler.handle(new DAGEventCommitCompleted(dagId, outputKey, false, t));
+ }
+ }
}
[16/50] [abbrv] tez git commit: TEZ-2333. Enable local fetch
optimization by default. (pramachandran)
Posted by ss...@apache.org.
TEZ-2333. Enable local fetch optimization by default. (pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/cbea5ad3
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/cbea5ad3
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/cbea5ad3
Branch: refs/heads/TEZ-2003
Commit: cbea5ad3a7500632d2c7586865174d6c9f4499ec
Parents: 8386cca
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Fri Apr 17 22:12:00 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Fri Apr 17 22:12:00 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/runtime/library/api/TezRuntimeConfiguration.java | 6 +-----
.../src/test/java/org/apache/tez/test/TestSecureShuffle.java | 1 +
3 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/cbea5ad3/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b0ce3cf..1888f15 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2333. Enable local fetch optimization by default.
TEZ-2310. Deadlock caused by StateChangeNotifier sending notifications on
thread holding locks
TEZ-1969. Stop the DAGAppMaster when a local mode client is stopped
http://git-wip-us.apache.org/repos/asf/tez/blob/cbea5ad3/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
index 9f11e4d..b1341f6 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
@@ -262,11 +262,7 @@ public class TezRuntimeConfiguration {
* If the shuffle input is on the local host bypass the http fetch and access the files directly
*/
public static final String TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH = TEZ_RUNTIME_PREFIX + "optimize.local.fetch";
-
- /**
- * local mode bypassing the http fetch is not enabled by default till we have unit tests in.
- */
- public static final boolean TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT = false;
+ public static final boolean TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT = true;
/**
* Expert level setting. Enable pipelined shuffle in ordered outputs and in unordered
http://git-wip-us.apache.org/repos/asf/tez/blob/cbea5ad3/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
index a0f7d8b..8da3f08 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
@@ -95,6 +95,7 @@ public class TestSecureShuffle {
new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
fs = miniDFSCluster.getFileSystem();
conf.set("fs.defaultFS", fs.getUri().toString());
+ conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, false);
}
@AfterClass
[26/50] [abbrv] tez git commit: TEZ-2340. TestRecoveryParser fails
(zjffdu)
Posted by ss...@apache.org.
TEZ-2340. TestRecoveryParser fails (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/d91eb283
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/d91eb283
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/d91eb283
Branch: refs/heads/TEZ-2003
Commit: d91eb283523a9dc98a73f79c845a4a7ecc8e5e16
Parents: 9faa54c
Author: Jeff Zhang <zj...@apache.org>
Authored: Thu Apr 23 08:53:18 2015 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Thu Apr 23 08:53:18 2015 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/tez/dag/app/TestRecoveryParser.java | 9 ++++++---
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/d91eb283/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5c91872..c32830e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2340. TestRecoveryParser fails
TEZ-2345. Tez UI: Enable cell level loading in all DAGs table
TEZ-2330. Create reconfigureVertex() API for input based initialization
TEZ-2292. Add e2e test for error reporting when vertex manager invokes
http://git-wip-us.apache.org/repos/asf/tez/blob/d91eb283/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
index 16c3a38..a256244 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
@@ -54,6 +54,7 @@ public class TestRecoveryParser {
private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR
+ TestRecoveryParser.class.getName() + "-tmpDir";
+ private ApplicationId appId;
private RecoveryParser parser;
private FileSystem localFS;
private Configuration conf;
@@ -65,7 +66,8 @@ public class TestRecoveryParser {
public void setUp() throws IllegalArgumentException, IOException {
this.conf = new Configuration();
this.localFS = FileSystem.getLocal(conf);
- this.recoveryPath = new Path(TEST_ROOT_DIR + "/recovery");
+ this.appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ this.recoveryPath = new Path(TEST_ROOT_DIR + "/" + appId + "/recovery");
this.localFS.delete(new Path(TEST_ROOT_DIR), true);
mockAppMaster = mock(DAGAppMaster.class);
mockAppMaster.dagNames = new HashSet<String>();
@@ -167,7 +169,7 @@ public class TestRecoveryParser {
// skipAllOtherEvents due to dag finished
@Test (timeout = 5000)
public void testSkipAllOtherEvents_2() throws IOException {
- ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1"));
@@ -205,7 +207,8 @@ public class TestRecoveryParser {
assertEquals(true, dagData.isCompleted);
// DAGSubmittedEvent, DAGInitializedEvent and DAGFinishedEvent is handled
verify(mockAppMaster).createDAG(any(DAGPlan.class),any(TezDAGID.class));
- verify(dagData.recoveredDAG).restoreFromEvent(isA(DAGInitializedEvent.class));
+ // DAGInitializedEvent may not been handled before DAGFinishedEvent,
+ // because DAGFinishedEvent's writeToRecoveryImmediately is true
verify(dagData.recoveredDAG).restoreFromEvent(isA(DAGFinishedEvent.class));
// DAGStartedEvent is skipped due to it is after DAGFinishedEvent
verify(dagData.recoveredDAG, never()).restoreFromEvent(isA(DAGStartedEvent.class));
[15/50] [abbrv] tez git commit: TEZ-2310. Deadlock caused by
StateChangeNotifier sending notifications on thread holding locks (bikas)
Posted by ss...@apache.org.
TEZ-2310. Deadlock caused by StateChangeNotifier sending notifications on thread holding locks (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8386cca0
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8386cca0
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8386cca0
Branch: refs/heads/TEZ-2003
Commit: 8386cca03ab89248765b2ac87dd78a50ca84ff3c
Parents: 3e6fc35
Author: Bikas Saha <bi...@apache.org>
Authored: Fri Apr 17 01:59:41 2015 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri Apr 17 01:59:41 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../java/org/apache/tez/dag/app/dag/DAG.java | 4 +
.../tez/dag/app/dag/StateChangeNotifier.java | 99 ++++++++++++++++++--
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 6 +-
.../tez/dag/app/dag/impl/VertexManager.java | 4 +-
.../dag/app/rm/TaskSchedulerEventHandler.java | 2 -
.../dag/app/dag/TestStateChangeNotifier.java | 57 +++++++++--
.../tez/dag/app/dag/impl/TestDAGImpl.java | 8 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 8 +-
9 files changed, 164 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/8386cca0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1a7609f..b0ce3cf 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,8 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2310. Deadlock caused by StateChangeNotifier sending notifications on
+ thread holding locks
TEZ-1969. Stop the DAGAppMaster when a local mode client is stopped
TEZ-714. OutputCommitters should not run in the main AM dispatcher thread
TEZ-2323. Fix TestOrderedWordcount to use MR memory configs.
http://git-wip-us.apache.org/repos/asf/tez/blob/8386cca0/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index 1b64754..4c3426a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.client.DAGStatusBuilder;
@@ -55,6 +56,9 @@ public interface DAG {
* @return job-counters and aggregate task-counters
*/
TezCounters getAllCounters();
+
+ @SuppressWarnings("rawtypes")
+ EventHandler getEventHandler();
/**
* Get Vertex by vertex name
http://git-wip-us.apache.org/repos/asf/tez/blob/8386cca0/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java
index d2b298b..260cbf3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java
@@ -22,19 +22,27 @@ package org.apache.tez.dag.app.dag;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.SetMultimap;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.event.VertexStateUpdate;
+import org.apache.tez.dag.app.dag.event.DAGEvent;
+import org.apache.tez.dag.app.dag.event.DAGEventType;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Tracks status updates from various components, and informs registered components about updates.
@@ -42,18 +50,88 @@ import org.apache.tez.dag.records.TezVertexID;
@InterfaceAudience.Private
public class StateChangeNotifier {
+ private static final Logger LOG = LoggerFactory.getLogger(StateChangeNotifier.class);
+
private final DAG dag;
private final SetMultimap<TezVertexID, ListenerContainer> vertexListeners;
private final ListMultimap<TezVertexID, VertexStateUpdate> lastKnowStatesMap;
private final ReentrantReadWriteLock listenersLock = new ReentrantReadWriteLock();
- private final ReentrantReadWriteLock.ReadLock readLock = listenersLock.readLock();
private final ReentrantReadWriteLock.WriteLock writeLock = listenersLock.writeLock();
+ BlockingQueue<NotificationEvent> eventQueue = new LinkedBlockingQueue<NotificationEvent>();
+ private Thread eventHandlingThread;
+ private volatile boolean stopEventHandling = false;
+
+ private static class NotificationEvent {
+ final VertexStateUpdate update;
+ final VertexStateUpdateListener listener;
+
+ public NotificationEvent(VertexStateUpdate update, VertexStateUpdateListener listener) {
+ this.update = update;
+ this.listener = listener;
+ }
+
+ void sentUpdate() {
+ listener.onStateUpdated(update);
+ }
+
+ @Override
+ public String toString() {
+ return "[ VertexState:(" + update + ") Listener:" + listener + " ]";
+ }
+ }
+
public StateChangeNotifier(DAG dag) {
this.dag = dag;
this.vertexListeners = Multimaps.synchronizedSetMultimap(
HashMultimap.<TezVertexID, ListenerContainer>create());
this.lastKnowStatesMap = LinkedListMultimap.create();
+ startThread();
+ }
+
+ private void startThread() {
+ this.eventHandlingThread = new Thread("State Change Notifier DAG: " + dag.getID()) {
+ @SuppressWarnings("unchecked")
+ @Override
+ public void run() {
+ while (!stopEventHandling && !Thread.currentThread().isInterrupted()) {
+ NotificationEvent event;
+ try {
+ event = eventQueue.take();
+ } catch (InterruptedException e) {
+ if(!stopEventHandling) {
+ LOG.warn("Continuing after interrupt : ", e);
+ }
+ continue;
+ }
+ try {
+ event.sentUpdate();
+ processedEventFromQueue();
+ } catch (Exception e) {
+ // TODO send user code exception - TEZ-2332
+ LOG.error("Error in state update notification for " + event, e);
+ dag.getEventHandler().handle(new DAGEvent(dag.getID(), DAGEventType.INTERNAL_ERROR));
+ return;
+ }
+ }
+ }
+ };
+ this.eventHandlingThread.setDaemon(true); // dont block exit on this
+ this.eventHandlingThread.start();
+ }
+
+ @VisibleForTesting
+ protected void processedEventFromQueue() {
+ }
+
+ @VisibleForTesting
+ protected void addedEventToQueue() {
+ }
+
+ public void stop() {
+ this.stopEventHandling = true;
+ if (eventHandlingThread != null)
+ eventHandlingThread.interrupt();
}
// -------------- VERTEX STATE CHANGE SECTION ---------------
@@ -99,14 +177,14 @@ public class StateChangeNotifier {
}
public void stateChanged(TezVertexID vertexId, VertexStateUpdate vertexStateUpdate) {
- readLock.lock();
+ writeLock.lock();
try {
lastKnowStatesMap.put(vertexId, vertexStateUpdate);
if (vertexListeners.containsKey(vertexId)) {
sendStateUpdate(vertexId, vertexStateUpdate);
}
} finally {
- readLock.unlock();
+ writeLock.unlock();
}
}
@@ -115,11 +193,18 @@ public class StateChangeNotifier {
for (ListenerContainer listenerContainer : vertexListeners.get(vertexId)) {
listenerContainer.sendStateUpdate(event);
}
-
}
-
- private static final class ListenerContainer {
+ private void enqueueNotification(NotificationEvent event) {
+ try {
+ eventQueue.put(event);
+ addedEventToQueue();
+ } catch (InterruptedException e) {
+ LOG.error("Failed to put event", e);
+ }
+ }
+
+ private final class ListenerContainer {
final VertexStateUpdateListener listener;
final Set<org.apache.tez.dag.api.event.VertexState> states;
@@ -135,7 +220,7 @@ public class StateChangeNotifier {
private void sendStateUpdate(VertexStateUpdate stateUpdate) {
if (states.contains(stateUpdate.getVertexState())) {
- listener.onStateUpdated(stateUpdate);
+ enqueueNotification(new NotificationEvent(stateUpdate, listener));
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/8386cca0/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 1c93dc6..9e55088 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -180,7 +180,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private final AppContext appContext;
private final UserGroupInformation dagUGI;
private final ACLManager aclManager;
- private final StateChangeNotifier entityUpdateTracker;
+ @VisibleForTesting
+ StateChangeNotifier entityUpdateTracker;
volatile Map<TezVertexID, Vertex> vertices = new HashMap<TezVertexID, Vertex>();
@VisibleForTesting
@@ -586,7 +587,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
return jobPlan;
}
- EventHandler getEventHandler() {
+ @Override
+ public EventHandler getEventHandler() {
return this.eventHandler;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/8386cca0/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
index 0be0aaa..bcea22c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
@@ -340,7 +340,9 @@ public class VertexManager {
}
@Override
- public synchronized void onStateUpdated(VertexStateUpdate event) {
+ public void onStateUpdated(VertexStateUpdate event) {
+ // this is not called by the vertex manager plugin.
+ // no need to synchronize this. similar to other external notification methods
enqueueAndScheduleNextEvent(new VertexManagerEventOnVertexStateUpdate(event));
}
http://git-wip-us.apache.org/repos/asf/tez/blob/8386cca0/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 0985d58..250446d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -19,8 +19,6 @@
package org.apache.tez.dag.app.rm;
import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/tez/blob/8386cca0/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java
index 6a505ef..e6d1c31 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java
@@ -30,18 +30,60 @@ import static org.mockito.Mockito.verify;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
-
+import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Lists;
+
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.event.VertexState;
import org.apache.tez.dag.api.event.VertexStateUpdate;
import org.apache.tez.dag.api.event.VertexStateUpdateParallelismUpdated;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezVertexID;
+import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
public class TestStateChangeNotifier {
+
+ // uses the thread based notification code path but effectively blocks update
+ // events till listeners have been notified
+ public static class StateChangeNotifierForTest extends StateChangeNotifier {
+ AtomicInteger count = new AtomicInteger(0);
+ AtomicInteger totalCount = new AtomicInteger(0);
+
+ public StateChangeNotifierForTest(DAG dag) {
+ super(dag);
+ }
+
+ public void reset() {
+ count.set(0);
+ totalCount.set(0);
+ }
+
+ @Override
+ protected void processedEventFromQueue() {
+ synchronized (count) {
+ if (count.decrementAndGet() == 0) {
+ count.notifyAll();
+ }
+ }
+ }
+
+ @Override
+ protected void addedEventToQueue() {
+ totalCount.incrementAndGet();
+ synchronized (count) {
+ // processing may finish by the time we get here
+ if (count.incrementAndGet() > 0) {
+ try {
+ count.wait();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+ }
@Test(timeout = 5000)
public void testEventsOnRegistration() {
@@ -51,7 +93,7 @@ public class TestStateChangeNotifier {
Vertex v3 = createMockVertex(dagId, 3);
DAG dag = createMockDag(dagId, v1, v2, v3);
- StateChangeNotifier tracker = new StateChangeNotifier(dag);
+ StateChangeNotifierForTest tracker = new StateChangeNotifierForTest(dag);
// Vertex has sent one event
notifyTracker(tracker, v1, VertexState.RUNNING);
@@ -72,7 +114,6 @@ public class TestStateChangeNotifier {
VertexState.SUCCEEDED), mockListener14);
ArgumentCaptor<VertexStateUpdate> argumentCaptor =
ArgumentCaptor.forClass(VertexStateUpdate.class);
-
verify(mockListener11, times(1)).onStateUpdated(argumentCaptor.capture());
assertEquals(VertexState.RUNNING,
argumentCaptor.getValue().getVertexState());
@@ -85,8 +126,10 @@ public class TestStateChangeNotifier {
verify(mockListener14, never()).onStateUpdated(any(VertexStateUpdate.class));
// Vertex has not notified of state
+ tracker.reset();
VertexStateUpdateListener mockListener2 = mock(VertexStateUpdateListener.class);
tracker.registerForVertexUpdates(v2.getName(), null, mockListener2);
+ Assert.assertEquals(0, tracker.totalCount.get()); // there should no be any event sent out
verify(mockListener2, never()).onStateUpdated(any(VertexStateUpdate.class));
// Vertex has notified about parallelism update only
@@ -104,7 +147,7 @@ public class TestStateChangeNotifier {
Vertex v1 = createMockVertex(dagId, 1);
DAG dag = createMockDag(dagId, v1);
- StateChangeNotifier tracker = new StateChangeNotifier(dag);
+ StateChangeNotifierForTest tracker = new StateChangeNotifierForTest(dag);
VertexStateUpdateListener mockListener = mock(VertexStateUpdateListener.class);
tracker.registerForVertexUpdates(v1.getName(), null, mockListener);
@@ -139,7 +182,7 @@ public class TestStateChangeNotifier {
Vertex v1 = createMockVertex(dagId, 1);
DAG dag = createMockDag(dagId, v1);
- StateChangeNotifier tracker = new StateChangeNotifier(dag);
+ StateChangeNotifierForTest tracker = new StateChangeNotifierForTest(dag);
VertexStateUpdateListener mockListener = mock(VertexStateUpdateListener.class);
tracker.registerForVertexUpdates(v1.getName(), null, mockListener);
@@ -157,7 +200,7 @@ public class TestStateChangeNotifier {
Vertex v1 = createMockVertex(dagId, 1);
DAG dag = createMockDag(dagId, v1);
- StateChangeNotifier tracker = new StateChangeNotifier(dag);
+ StateChangeNotifierForTest tracker = new StateChangeNotifierForTest(dag);
VertexStateUpdateListener mockListener = mock(VertexStateUpdateListener.class);
tracker.registerForVertexUpdates(v1.getName(), EnumSet.of(
@@ -199,7 +242,7 @@ public class TestStateChangeNotifier {
Vertex v1 = createMockVertex(dagId, 1);
DAG dag = createMockDag(dagId, v1);
- StateChangeNotifier tracker = new StateChangeNotifier(dag);
+ StateChangeNotifierForTest tracker = new StateChangeNotifierForTest(dag);
VertexStateUpdateListener mockListener = mock(VertexStateUpdateListener.class);
tracker.registerForVertexUpdates(v1.getName(), null, mockListener);
http://git-wip-us.apache.org/repos/asf/tez/blob/8386cca0/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 7e944ef..228d6b8 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -33,9 +33,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
-import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
@@ -46,7 +44,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -97,6 +94,7 @@ import org.apache.tez.dag.app.dag.DAGTerminationCause;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.app.dag.TaskAttemptStateInternal;
+import org.apache.tez.dag.app.dag.TestStateChangeNotifier.StateChangeNotifierForTest;
import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.VertexState;
import org.apache.tez.dag.app.dag.VertexTerminationCause;
@@ -786,6 +784,7 @@ public class TestDAGImpl {
dag = new DAGImpl(dagId, conf, dagPlan,
dispatcher.getEventHandler(), taskAttemptListener,
fsTokens, clock, "user", thh, appContext);
+ dag.entityUpdateTracker = new StateChangeNotifierForTest(dag);
doReturn(dag).when(appContext).getCurrentDAG();
mrrAppContext = mock(AppContext.class);
doReturn(aclManager).when(mrrAppContext).getAMACLManager();
@@ -796,6 +795,7 @@ public class TestDAGImpl {
dispatcher.getEventHandler(), taskAttemptListener,
fsTokens, clock, "user", thh,
mrrAppContext);
+ mrrDag.entityUpdateTracker = new StateChangeNotifierForTest(mrrDag);
doReturn(conf).when(mrrAppContext).getAMConf();
doReturn(mrrDag).when(mrrAppContext).getCurrentDAG();
doReturn(appAttemptId).when(mrrAppContext).getApplicationAttemptId();
@@ -810,6 +810,7 @@ public class TestDAGImpl {
dispatcher.getEventHandler(), taskAttemptListener,
fsTokens, clock, "user", thh,
groupAppContext);
+ groupDag.entityUpdateTracker = new StateChangeNotifierForTest(groupDag);
doReturn(conf).when(groupAppContext).getAMConf();
doReturn(groupDag).when(groupAppContext).getCurrentDAG();
doReturn(appAttemptId).when(groupAppContext).getApplicationAttemptId();
@@ -858,6 +859,7 @@ public class TestDAGImpl {
dagWithCustomEdge = new DAGImpl(dagWithCustomEdgeId, conf, dagPlanWithCustomEdge,
dispatcher.getEventHandler(), taskAttemptListener,
fsTokens, clock, "user", thh, dagWithCustomEdgeAppContext);
+ dagWithCustomEdge.entityUpdateTracker = new StateChangeNotifierForTest(dagWithCustomEdge);
doReturn(conf).when(dagWithCustomEdgeAppContext).getAMConf();
doReturn(execService).when(dagWithCustomEdgeAppContext).getExecService();
doReturn(dagWithCustomEdge).when(dagWithCustomEdgeAppContext).getCurrentDAG();
http://git-wip-us.apache.org/repos/asf/tez/blob/8386cca0/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 891da23..c752965 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -114,6 +114,7 @@ import org.apache.tez.dag.app.dag.RootInputInitializerManager;
import org.apache.tez.dag.app.dag.StateChangeNotifier;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttemptStateInternal;
+import org.apache.tez.dag.app.dag.TestStateChangeNotifier.StateChangeNotifierForTest;
import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.VertexState;
import org.apache.tez.dag.app.dag.VertexStateUpdateListener;
@@ -221,7 +222,7 @@ public class TestVertexImpl {
private VertexEventDispatcher vertexEventDispatcher;
private DagEventDispatcher dagEventDispatcher;
private HistoryEventHandler historyEventHandler;
- private StateChangeNotifier updateTracker;
+ private StateChangeNotifierForTest updateTracker;
private static TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption;
public static class CountingOutputCommitter extends OutputCommitter {
@@ -2174,7 +2175,7 @@ public class TestVertexImpl {
for (PlanVertexGroupInfo groupInfo : dagPlan.getVertexGroupsList()) {
vertexGroups.put(groupInfo.getGroupName(), new VertexGroupInfo(groupInfo));
}
- updateTracker = new StateChangeNotifier(appContext.getCurrentDAG());
+ updateTracker = new StateChangeNotifierForTest(appContext.getCurrentDAG());
setupVertices();
when(dag.getVertex(any(TezVertexID.class))).thenAnswer(new Answer<Vertex>() {
@Override
@@ -2243,6 +2244,7 @@ public class TestVertexImpl {
@After
public void teardown() {
+ updateTracker.stop();
if (dispatcher.isInState(STATE.STARTED)) {
dispatcher.await();
dispatcher.stop();
@@ -3734,7 +3736,6 @@ public class TestVertexImpl {
v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
}
dispatcher.await();
-
Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
// At this point, 3 events should have been received - since the dispatcher is complete.
@@ -4108,7 +4109,6 @@ public class TestVertexImpl {
while (v3.getState() != VertexState.RUNNING) {
Thread.sleep(10);
}
-
Assert.assertEquals(VertexState.RUNNING, v3.getState());
// Events should have been cleared from the vertex.
Assert.assertEquals(0, v3.pendingInitializerEvents.size());
[24/50] [abbrv] tez git commit: TEZ-2330. Create reconfigureVertex()
API for input based initialization (bikas)
Posted by ss...@apache.org.
TEZ-2330. Create reconfigureVertex() API for input based initialization (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ec45c510
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ec45c510
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ec45c510
Branch: refs/heads/TEZ-2003
Commit: ec45c510c04eead59799813521f5ce0c6868960f
Parents: c6e400e
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Apr 21 19:24:00 2015 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Apr 21 19:24:00 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/dag/api/VertexManagerPluginContext.java | 26 ++++++++++++++++++++
.../java/org/apache/tez/dag/app/dag/Vertex.java | 4 +++
.../app/dag/impl/RootInputVertexManager.java | 9 +++++--
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 12 ++++++---
.../tez/dag/app/dag/impl/VertexManager.java | 12 +++++++++
.../tez/dag/app/dag/impl/TestVertexImpl.java | 2 +-
7 files changed, 59 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/ec45c510/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0c83c08..35cf312 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2330. Create reconfigureVertex() API for input based initialization
TEZ-2292. Add e2e test for error reporting when vertex manager invokes
plugin APIs
TEZ-2308. Add set/get of record counts in task/vertex statistics
http://git-wip-us.apache.org/repos/asf/tez/blob/ec45c510/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
index 8b0e89e..345ea43 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
@@ -147,6 +147,7 @@ public interface VertexManagerPluginContext {
* @param rootInputSpecUpdate Updated Root Input specifications, if any.
* If none specified, a default of 1 physical input is used
*/
+ @Deprecated
public void setVertexParallelism(int parallelism,
@Nullable VertexLocationHint locationHint,
@Nullable Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
@@ -178,6 +179,31 @@ public interface VertexManagerPluginContext {
@Nullable Map<String, EdgeProperty> sourceEdgeProperties);
/**
+ * API to reconfigure a {@link Vertex} that is reading root inputs based on
+ * the data read from the root inputs. Root inputs are external data sources
+ * that provide the initial data for the DAG and are added to the
+ * {@link Vertex} using the
+ * {@link Vertex#addDataSource(String, DataSourceDescriptor)} API. Typically,
+ * the parallelism of such vertices is determined at runtime by gathering
+ * information about the data source. This API may be used to set the
+ * parallelism of the vertex at runtime based on the data sources, as well as
+ * changing the specification for those inputs.
+ * @param rootInputSpecUpdate
+ * The key of the map is the name of the data source and the value is
+ * the updated {@link InputSpecUpdate} for that data source. If none
+ * specified, a default value is used. See {@link InputSpecUpdate}
+ * for details.
+ * @param locationHint
+ * the placement policy for tasks specified at
+ * {@link VertexLocationHint}s
+ * @param parallelism
+ * New number of tasks in the vertex
+ */
+ public void reconfigureVertex(@Nullable Map<String, InputSpecUpdate> rootInputSpecUpdate,
+ @Nullable VertexLocationHint locationHint,
+ int parallelism);
+
+ /**
* Allows a VertexManagerPlugin to assign Events for Root Inputs
*
* For regular Event Routing changes - the EdgeManager should be configured
http://git-wip-us.apache.org/repos/asf/tez/blob/ec45c510/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index 6c85b85..77ef6e0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -101,6 +101,10 @@ public interface Vertex extends Comparable<Vertex> {
public void reconfigureVertex(int parallelism,
@Nullable VertexLocationHint locationHint,
@Nullable Map<String, EdgeProperty> sourceEdgeProperties) throws AMUserCodeException;
+
+ public void reconfigureVertex(@Nullable Map<String, InputSpecUpdate> rootInputSpecUpdate,
+ int parallelism,
+ @Nullable VertexLocationHint locationHint) throws AMUserCodeException;
void setVertexLocationHint(VertexLocationHint vertexLocationHint);
void vertexReconfigurationPlanned();
http://git-wip-us.apache.org/repos/asf/tez/blob/ec45c510/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
index e850286..c1e96f3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
@@ -30,12 +30,17 @@ import org.apache.tez.runtime.api.InputSpecUpdate;
import org.apache.tez.runtime.api.events.InputConfigureVertexTasksEvent;
import org.apache.tez.runtime.api.events.InputDataInformationEvent;
import org.apache.tez.runtime.api.events.InputUpdatePayloadEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
public class RootInputVertexManager extends ImmediateStartVertexManager {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RootInputVertexManager.class);
+
private String configuredInputName;
public RootInputVertexManager(VertexManagerPluginContext context) {
@@ -66,8 +71,8 @@ public class RootInputVertexManager extends ImmediateStartVertexManager {
inputName,
cEvent.getInputSpecUpdate() == null ? InputSpecUpdate
.getDefaultSinglePhysicalInputSpecUpdate() : cEvent.getInputSpecUpdate());
- getContext().setVertexParallelism(cEvent.getNumTasks(),
- cEvent.getLocationHint(), null, rootInputSpecUpdate);
+ getContext().reconfigureVertex(rootInputSpecUpdate, cEvent.getLocationHint(),
+ cEvent.getNumTasks());
}
if (event instanceof InputUpdatePayloadEvent) {
// No tasks should have been started yet. Checked by initial state check.
http://git-wip-us.apache.org/repos/asf/tez/blob/ec45c510/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 5dfcb8e..e22343b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -24,7 +24,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
@@ -82,7 +81,6 @@ import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
import org.apache.tez.dag.api.client.ProgressBuilder;
import org.apache.tez.dag.api.client.StatusGetOpts;
import org.apache.tez.dag.api.client.VertexStatus;
-import org.apache.tez.dag.api.client.VertexStatus.State;
import org.apache.tez.dag.api.client.VertexStatusBuilder;
import org.apache.tez.dag.api.event.VertexStateUpdate;
import org.apache.tez.dag.api.event.VertexStateUpdateParallelismUpdated;
@@ -96,7 +94,6 @@ import org.apache.tez.dag.app.ContainerContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.dag.DAGState;
import org.apache.tez.dag.app.dag.RootInputInitializerManager;
import org.apache.tez.dag.app.dag.StateChangeNotifier;
import org.apache.tez.dag.app.dag.Task;
@@ -1433,7 +1430,14 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@Nullable Map<String, EdgeProperty> sourceEdgeProperties) throws AMUserCodeException {
setParallelism(parallelism, locationHint, sourceEdgeProperties, null, false, true);
}
-
+
+ @Override
+ public void reconfigureVertex(@Nullable Map<String, InputSpecUpdate> rootInputSpecUpdate,
+ int parallelism,
+ @Nullable VertexLocationHint locationHint) throws AMUserCodeException {
+ setParallelism(parallelism, locationHint, null, rootInputSpecUpdate, false, true);
+ }
+
@Override
public void setParallelism(int parallelism, VertexLocationHint vertexLocationHint,
Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
http://git-wip-us.apache.org/repos/asf/tez/blob/ec45c510/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
index 2ac1acf..1ed42fc 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
@@ -185,6 +185,18 @@ public class VertexManager {
throw new TezUncheckedException(e);
}
}
+
+ @Override
+ public void reconfigureVertex(@Nullable Map<String, InputSpecUpdate> rootInputSpecUpdate,
+ @Nullable VertexLocationHint locationHint,
+ int parallelism) {
+ checkAndThrowIfDone();
+ try {
+ managedVertex.reconfigureVertex(rootInputSpecUpdate, parallelism, locationHint);
+ } catch (AMUserCodeException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
@Override
public synchronized void scheduleVertexTasks(List<TaskWithLocationHint> tasks) {
http://git-wip-us.apache.org/repos/asf/tez/blob/ec45c510/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 2403599..3147093 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -5566,7 +5566,7 @@ public class TestVertexImpl {
}
map.put("input4", InputSpecUpdate.createPerTaskInputSpecUpdate(pInputList));
}
- getContext().setVertexParallelism(NUM_TASKS, null, null, map);
+ getContext().reconfigureVertex(map, null, NUM_TASKS);
}
}
[08/50] [abbrv] tez git commit: TEZ-714. OutputCommitters should not
run in the main AM dispatcher thread (zjffdu)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java
new file mode 100644
index 0000000..fe7984b
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.mapreduce.examples;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.DataSinkDescriptor;
+import org.apache.tez.dag.api.GroupInputEdge;
+import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.VertexGroup;
+import org.apache.tez.examples.TezExampleBase;
+import org.apache.tez.mapreduce.output.MROutput;
+import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfig;
+import org.apache.tez.runtime.library.input.ConcatenatedMergedKeyValuesInput;
+import org.apache.tez.runtime.library.partitioner.HashPartitioner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * v1 -> v3 <br/>
+ * v2 -> v3 <br/>
+ * (v1,v2) is connected to v3 as vertex group. <br/>
+ * (v1,v2) have multiple shared outputs, each of them have its own multiple outputs.
+ * And v3 also has multiple outputs. </br>
+ */
+public class MultipleCommitsExample extends TezExampleBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MultipleCommitsExample.class);
+ private static final String UV12OutputNamePrefix = "uv12Output";
+ private static final String V1OutputNamePrefix = "v1Output";
+ private static final String V2OutputNamePrefix = "v2Output";
+ private static final String V3OutputNamePrefix = "v3Output";
+
+ public static final String CommitOnVertexSuccessOption = "commitOnVertexSuccess";
+
+ @Override
+ protected void printUsage() {
+ System.err.println("Usage: "
+ + " multiplecommitsExample v1OutputPrefix v1OutputNum v2OutputPrefix v2OutputNum"
+ + " uv12OutputPrefix uv12OutputNum v3OutputPrefix v3OutputNum"
+ + " [" + CommitOnVertexSuccessOption + "]" + "(default false)");
+ }
+
+ @Override
+ protected int validateArgs(String[] otherArgs) {
+ if (otherArgs.length != 8 && otherArgs.length != 9) {
+ return 2;
+ }
+ if (otherArgs.length == 9 && !otherArgs[8].equals(CommitOnVertexSuccessOption)) {
+ return 2;
+ }
+ return 0;
+ }
+
+ public static class MultipleOutputProcessor extends SimpleMRProcessor {
+
+ MultipleOutputProcessorConfig config;
+
+ public MultipleOutputProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void initialize() throws Exception {
+ super.initialize();
+ config = MultipleOutputProcessorConfig.fromUserPayload(getContext().getUserPayload());
+ }
+
+ @Override
+ public void run() throws Exception {
+ for (int i=0;i < config.outputNum;++i) {
+ KeyValueWriter writer = (KeyValueWriter)
+ getOutputs().get(config.outputNamePrefix+"_" + i).getWriter();
+ writer.write(NullWritable.get(), new Text("dummy"));
+ }
+ for (int i=0;i < config.sharedOutputNum; ++i) {
+ KeyValueWriter writer = (KeyValueWriter)
+ getOutputs().get(config.sharedOutputNamePrefix +"_" + i).getWriter();
+ writer.write(NullWritable.get(), new Text("dummy"));
+ }
+ }
+
+ public static class MultipleOutputProcessorConfig implements Writable {
+
+ String outputNamePrefix;
+ int outputNum;
+ String sharedOutputNamePrefix = null;
+ int sharedOutputNum;
+
+ public MultipleOutputProcessorConfig(){
+
+ }
+
+ public MultipleOutputProcessorConfig(String outputNamePrefix, int outputNum) {
+ this.outputNamePrefix = outputNamePrefix;
+ this.outputNum = outputNum;
+ }
+
+ public MultipleOutputProcessorConfig(String outputNamePrefix, int outputNum,
+ String sharedOutputNamePrefix, int sharedOutputNum) {
+ this.outputNamePrefix = outputNamePrefix;
+ this.outputNum = outputNum;
+ this.sharedOutputNamePrefix = sharedOutputNamePrefix;
+ this.sharedOutputNum = sharedOutputNum;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ new Text(outputNamePrefix).write(out);
+ out.writeInt(outputNum);
+ if (sharedOutputNamePrefix != null) {
+ new BooleanWritable(true).write(out);
+ new Text(sharedOutputNamePrefix).write(out);
+ out.writeInt(sharedOutputNum);
+ } else {
+ new BooleanWritable(false).write(out);
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ Text outputNameText = new Text();
+ outputNameText.readFields(in);
+ outputNamePrefix = outputNameText.toString();
+ outputNum = in.readInt();
+ BooleanWritable hasSharedOutputs = new BooleanWritable();
+ hasSharedOutputs.readFields(in);
+ if (hasSharedOutputs.get()) {
+ Text sharedOutputNamePrefixText = new Text();
+ sharedOutputNamePrefixText.readFields(in);
+ sharedOutputNamePrefix = sharedOutputNamePrefixText.toString();
+ sharedOutputNum = in.readInt();
+ }
+ }
+
+ public UserPayload toUserPayload() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ this.write(new DataOutputStream(out));
+ return UserPayload.create(ByteBuffer.wrap(out.toByteArray()));
+ }
+
+ public static MultipleOutputProcessorConfig fromUserPayload(UserPayload payload)
+ throws IOException {
+ MultipleOutputProcessorConfig config = new MultipleOutputProcessorConfig();
+ config.readFields(new DataInputStream(
+ new ByteArrayInputStream(payload.deepCopyAsArray())));
+ return config;
+ }
+ }
+ }
+
+ @Override
+ protected int runJob(String[] args, TezConfiguration tezConf,
+ TezClient tezClient) throws Exception {
+ boolean commitOnVertexSuccess =
+ args.length == 5 && args[4].equals(CommitOnVertexSuccessOption) ? true : false;
+ DAG dag = createDAG(tezConf, args[0], Integer.parseInt(args[1]),
+ args[2], Integer.parseInt(args[3]),
+ args[4], Integer.parseInt(args[5]),
+ args[6], Integer.parseInt(args[7]),
+ commitOnVertexSuccess);
+ LOG.info("Running MultipleCommitsExample");
+ return runDag(dag, false, LOG);
+ }
+
+ private DAG createDAG(TezConfiguration tezConf,
+ String v1OutputPathPrefix, int v1OutputNum, String v2OutputPathPrefix, int v2OutputNum,
+ String uv12OutputPathPrefix, int uv12OutputNum,
+ String v3OutputPathPrefix, int v3OutputNum, boolean commitOnVertexSuccess) throws IOException {
+ DAG dag = DAG.create("multipleCommitsDAG");
+ dag.setConf(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, !commitOnVertexSuccess + "");
+ Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(MultipleOutputProcessor.class.getName())
+ .setUserPayload(
+ new MultipleOutputProcessor.MultipleOutputProcessorConfig(
+ V1OutputNamePrefix, v1OutputNum, UV12OutputNamePrefix, uv12OutputNum)
+ .toUserPayload()), 2);
+ Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create(MultipleOutputProcessor.class.getName())
+ .setUserPayload(
+ new MultipleOutputProcessor.MultipleOutputProcessorConfig(
+ V2OutputNamePrefix, v2OutputNum, UV12OutputNamePrefix, uv12OutputNum)
+ .toUserPayload()), 2);
+ // add data sinks for v1
+ for (int i=0;i<v1OutputNum;++i) {
+ DataSinkDescriptor sink = MROutput.createConfigBuilder(
+ new Configuration(tezConf), TextOutputFormat.class, v1OutputPathPrefix + "_" + i).build();
+ v1.addDataSink(V1OutputNamePrefix + "_" + i, sink);
+ }
+ // add data sinks for v2
+ for (int i=0;i<v2OutputNum;++i) {
+ DataSinkDescriptor sink = MROutput.createConfigBuilder(
+ new Configuration(tezConf), TextOutputFormat.class, v2OutputPathPrefix + "_" + i).build();
+ v2.addDataSink(V2OutputNamePrefix + "_" + i, sink);
+ }
+ // add data sinks for (v1,v2)
+ VertexGroup uv12 = dag.createVertexGroup("uv12", v1,v2);
+ for (int i=0;i<uv12OutputNum;++i) {
+ DataSinkDescriptor sink = MROutput.createConfigBuilder(
+ new Configuration(tezConf), TextOutputFormat.class, uv12OutputPathPrefix + "_" + i).build();
+ uv12.addDataSink(UV12OutputNamePrefix + "_" + i, sink);
+ }
+
+ Vertex v3 = Vertex.create("v3", ProcessorDescriptor.create(MultipleOutputProcessor.class.getName())
+ .setUserPayload(
+ new MultipleOutputProcessor.MultipleOutputProcessorConfig(V3OutputNamePrefix, v3OutputNum)
+ .toUserPayload()), 2);
+ // add data sinks for v3
+ for (int i=0;i<v3OutputNum;++i) {
+ DataSinkDescriptor sink = MROutput.createConfigBuilder(
+ new Configuration(tezConf), TextOutputFormat.class, v3OutputPathPrefix + "_" + i).build();
+ v3.addDataSink(V3OutputNamePrefix + "_" + i, sink);
+ }
+
+ OrderedPartitionedKVEdgeConfig edgeConfig =
+ OrderedPartitionedKVEdgeConfig.newBuilder(
+ NullWritable.class.getName(), Text.class.getName(), HashPartitioner.class.getName())
+ .setFromConfiguration(tezConf)
+ .build();
+ GroupInputEdge edge = GroupInputEdge.create(uv12, v3, edgeConfig.createDefaultEdgeProperty(),
+ InputDescriptor.create(
+ ConcatenatedMergedKeyValuesInput.class.getName()));
+ dag.addVertex(v1)
+ .addVertex(v2)
+ .addVertex(v3)
+ .addEdge(edge);
+ return dag;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new MultipleCommitsExample(), args);
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d932579b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
index 3fe249c..db212fa 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
@@ -33,12 +33,14 @@ import java.util.BitSet;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
+import java.util.Random;
import java.util.Set;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -72,6 +74,7 @@ import org.apache.tez.examples.JoinDataGen;
import org.apache.tez.examples.HashJoinExample;
import org.apache.tez.examples.JoinValidate;
import org.apache.tez.examples.SortMergeJoinExample;
+import org.apache.tez.mapreduce.examples.MultipleCommitsExample;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.InputInitializer;
import org.apache.tez.runtime.api.InputInitializerContext;
@@ -820,6 +823,81 @@ public class TestTezJobs {
}
}
+ @Test(timeout = 60000)
+ public void testMultipleCommits_OnDAGSuccess() throws Exception {
+ Path stagingDirPath = new Path("/tmp/commit-staging-dir");
+ Random rand = new Random();
+ String v1OutputPathPrefix = "/tmp/commit-output-v1";
+ int v1OutputNum = rand.nextInt(10) + 1;
+ String v2OutputPathPrefix = "/tmp/commit-output-v2";
+ int v2OutputNum = rand.nextInt(10) + 1;
+ String uv12OutputPathPrefix = "/tmp/commit-output-uv12";
+ int uv12OutputNum = rand.nextInt(10) + 1;
+ String v3OutputPathPrefix = "/tmp/commit-output-v3";
+ int v3OutputNum = rand.nextInt(10) + 1;
+ TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
+ TezClient tezSession = null;
+
+ try {
+ MultipleCommitsExample job = new MultipleCommitsExample();
+ Assert.assertTrue("MultipleCommitsExample failed", job.run(tezConf,
+ new String[]{ v1OutputPathPrefix, v1OutputNum + "", v2OutputPathPrefix, v2OutputNum + "",
+ uv12OutputPathPrefix, uv12OutputNum + "", v3OutputPathPrefix, v3OutputNum + ""}, null)==0);
+ verifyCommits(v1OutputPathPrefix, v1OutputNum);
+ verifyCommits(v2OutputPathPrefix, v2OutputNum);
+ verifyCommits(uv12OutputPathPrefix, uv12OutputNum);
+ verifyCommits(v3OutputPathPrefix, v3OutputNum);
+ } finally {
+ remoteFs.delete(stagingDirPath, true);
+ if (tezSession != null) {
+ tezSession.stop();
+ }
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testMultipleCommits_OnVertexSuccess() throws Exception {
+ Path stagingDirPath = new Path("/tmp/commit-staging-dir");
+ Random rand = new Random();
+ String v1OutputPathPrefix = "/tmp/commit-output-v1";
+ int v1OutputNum = rand.nextInt(10) + 1;
+ String v2OutputPathPrefix = "/tmp/commit-output-v2";
+ int v2OutputNum = rand.nextInt(10) + 1;
+ String uv12OutputPathPrefix = "/tmp/commit-output-uv12";
+ int uv12OutputNum = rand.nextInt(10) + 1;
+ String v3OutputPathPrefix = "/tmp/commit-output-v3";
+ int v3OutputNum = rand.nextInt(10) + 1;
+ TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
+ TezClient tezSession = null;
+
+ try {
+ MultipleCommitsExample job = new MultipleCommitsExample();
+ Assert.assertTrue("MultipleCommitsExample failed", job.run(tezConf,
+ new String[]{ v1OutputPathPrefix, v1OutputNum + "", v2OutputPathPrefix, v2OutputNum + "",
+ uv12OutputPathPrefix, uv12OutputNum + "", v3OutputPathPrefix, v3OutputNum + "",
+ MultipleCommitsExample.CommitOnVertexSuccessOption}, null)==0);
+ verifyCommits(v1OutputPathPrefix, v1OutputNum);
+ verifyCommits(v2OutputPathPrefix, v2OutputNum);
+ verifyCommits(uv12OutputPathPrefix, uv12OutputNum);
+ verifyCommits(v3OutputPathPrefix, v3OutputNum);
+ } finally {
+ remoteFs.delete(stagingDirPath, true);
+ if (tezSession != null) {
+ tezSession.stop();
+ }
+ }
+ }
+
+ private void verifyCommits(String outputPrefix, int outputNum) throws IllegalArgumentException, IOException {
+ for (int i=0; i< outputNum; ++i) {
+ String outputDir = outputPrefix + "_" + i;
+ Assert.assertTrue("Output of " + outputDir + " is not succeeded",
+ remoteFs.exists(new Path( outputDir + "/_SUCCESS")));
+ }
+ }
+
private static final String VERTEX_WITH_INITIALIZER_NAME = "VertexWithInitializer";
private static final String EVENT_GENERATING_VERTEX_NAME = "EventGeneratingVertex";
private static final String INPUT1_NAME = "Input1";
[43/50] [abbrv] tez git commit: TEZ-2241. Miscellaneous fixes after
last reabse. (sseth)
Posted by ss...@apache.org.
TEZ-2241. Miscellaneous fixes after last reabse. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b828d156
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b828d156
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b828d156
Branch: refs/heads/TEZ-2003
Commit: b828d1561784ee2fd2a101ac1e98f05bf00795ec
Parents: e901aeb
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Mar 26 17:57:55 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 23 16:47:33 2015 -0700
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java | 2 ++
.../org/apache/tez/dag/app/TezLocalTaskCommunicatorImpl.java | 6 +++---
.../java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java | 6 +++---
.../apache/tez/dag/app/launcher/ContainerLauncherRouter.java | 6 +++---
.../test/java/org/apache/tez/dag/app/MockDAGAppMaster.java | 2 +-
tez-ext-service-tests/pom.xml | 4 ++--
.../tez/dag/app/launcher/TezTestServiceContainerLauncher.java | 6 +++---
.../dag/app/launcher/TezTestServiceNoOpContainerLauncher.java | 6 +++---
.../tez/dag/app/rm/TezTestServiceTaskSchedulerService.java | 7 ++++---
.../dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java | 7 ++++---
.../org/apache/tez/service/MiniTezTestServiceCluster.java | 6 +++---
.../java/org/apache/tez/service/impl/ContainerRunnerImpl.java | 5 +++--
.../test/java/org/apache/tez/service/impl/TezTestService.java | 4 ++--
.../tez/service/impl/TezTestServiceProtocolServerImpl.java | 6 +++---
.../org/apache/tez/shufflehandler/FadvisedChunkedFile.java | 6 +++---
.../org/apache/tez/shufflehandler/FadvisedFileRegion.java | 6 +++---
.../test/java/org/apache/tez/shufflehandler/IndexCache.java | 6 +++---
.../java/org/apache/tez/shufflehandler/ShuffleHandler.java | 6 +++---
.../java/org/apache/tez/tests/TestExternalTezServices.java | 6 +++---
20 files changed, 55 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 774a685..d51686d 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -11,5 +11,6 @@ ALL CHANGES:
TEZ-2139. Update tez version to 0.7.0-TEZ-2003-SNAPSHOT.
TEZ-2175. Task priority should be available to the TaskCommunicator plugin.
TEZ-2187. Allow TaskCommunicators to report failed / killed attempts.
+ TEZ-2241. Miscellaneous fixes after last reabse.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 9f33a1f..bea0902 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -56,6 +56,8 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.runtime.api.impl.TezEvent;
import org.apache.tez.common.security.JobTokenSecretManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-dag/src/main/java/org/apache/tez/dag/app/TezLocalTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezLocalTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezLocalTaskCommunicatorImpl.java
index 3704cc4..47688d1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezLocalTaskCommunicatorImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezLocalTaskCommunicatorImpl.java
@@ -18,14 +18,14 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.tez.dag.api.TaskCommunicatorContext;
import org.apache.tez.dag.api.TezUncheckedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TezLocalTaskCommunicatorImpl extends TezTaskCommunicatorImpl {
- private static final Log LOG = LogFactory.getLog(TezLocalTaskCommunicatorImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TezLocalTaskCommunicatorImpl.class);
public TezLocalTaskCommunicatorImpl(
TaskCommunicatorContext taskCommunicatorContext) {
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
index f288748..9d0c031 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
@@ -23,8 +23,6 @@ import java.util.concurrent.ConcurrentMap;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -55,11 +53,13 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.runtime.api.impl.TaskSpec;
import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public class TezTaskCommunicatorImpl extends TaskCommunicator {
- private static final Log LOG = LogFactory.getLog(TezTaskCommunicatorImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TezTaskCommunicatorImpl.class);
private static final ContainerTask TASK_FOR_INVALID_JVM = new ContainerTask(
null, true, null, null, false);
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
index 70b0cbc..dd3571e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
@@ -19,8 +19,6 @@ import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -30,11 +28,13 @@ import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ContainerLauncherRouter extends AbstractService
implements EventHandler<NMCommunicatorEvent> {
- static final Log LOG = LogFactory.getLog(ContainerLauncherImpl.class);
+ static final Logger LOG = LoggerFactory.getLogger(ContainerLauncherImpl.class);
private final ContainerLauncher containerLaunchers[];
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index 73e93c5..d40b78d 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -336,7 +336,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
long startTime = System.nanoTime();
long startCpuTime = threadMxBean.getCurrentThreadCpuTime();
TaskHeartbeatResponse response = taListener.heartbeat(request);
- if (response.shouldDie()) {
+ if (response.isShouldDie()) {
cData.remove();
} else {
cData.nextFromEventId += response.getEvents().size();
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml
index d64dc58..885cb45 100644
--- a/tez-ext-service-tests/pom.xml
+++ b/tez-ext-service-tests/pom.xml
@@ -28,8 +28,8 @@
<dependencies>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
index cb6308c..9d22196 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
@@ -18,8 +18,6 @@ import java.net.InetSocketAddress;
import com.google.common.base.Preconditions;
import com.google.protobuf.ByteString;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -40,6 +38,8 @@ import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
import org.apache.tez.service.TezTestServiceConfConstants;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TezTestServiceContainerLauncher extends AbstractService implements ContainerLauncher {
@@ -47,7 +47,7 @@ public class TezTestServiceContainerLauncher extends AbstractService implements
// TODO May need multiple connections per target machine, depending upon how synchronization is handled in the RPC layer
- static final Log LOG = LogFactory.getLog(TezTestServiceContainerLauncher.class);
+ static final Logger LOG = LoggerFactory.getLogger(TezTestServiceContainerLauncher.class);
private final AppContext context;
private final String tokenIdentifier;
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java
index 8c8e486..977d0d3 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java
@@ -14,8 +14,6 @@
package org.apache.tez.dag.app.launcher;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.util.Clock;
@@ -28,10 +26,12 @@ import org.apache.tez.dag.app.rm.container.AMContainerEventLaunched;
import org.apache.tez.dag.app.rm.container.AMContainerEventType;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TezTestServiceNoOpContainerLauncher extends AbstractService implements ContainerLauncher {
- static final Log LOG = LogFactory.getLog(TezTestServiceNoOpContainerLauncher.class);
+ static final Logger LOG = LoggerFactory.getLogger(TezTestServiceNoOpContainerLauncher.class);
private final AppContext context;
private final Clock clock;
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
index 872d592..50dfb24 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
@@ -28,8 +28,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -41,11 +39,14 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.dag.api.TaskAttemptEndReason;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.service.TezTestServiceConfConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
- private static final Log LOG = LogFactory.getLog(TezTestServiceTaskSchedulerService.class);
+ private static final Logger
+ LOG = LoggerFactory.getLogger(TezTestServiceTaskSchedulerService.class);
private final ExecutorService appCallbackExecutor;
private final TaskSchedulerAppCallback appClientDelegate;
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
index e3385a2..ef983c2 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
@@ -23,8 +23,6 @@ import java.util.concurrent.RejectedExecutionException;
import com.google.protobuf.ByteString;
import com.google.protobuf.ServiceException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.ipc.RemoteException;
@@ -41,11 +39,14 @@ import org.apache.tez.runtime.api.impl.TaskSpec;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto;
import org.apache.tez.util.ProtoConverters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TezTestServiceTaskCommunicatorImpl extends TezTaskCommunicatorImpl {
- private static final Log LOG = LogFactory.getLog(TezTestServiceTaskCommunicatorImpl.class);
+ private static final Logger
+ LOG = LoggerFactory.getLogger(TezTestServiceTaskCommunicatorImpl.class);
private final TezTestServiceCommunicator communicator;
private final SubmitWorkRequestProto BASE_SUBMIT_WORK_REQUEST;
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
index 0ac0b33..c5ff02d 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
@@ -19,8 +19,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import com.google.common.base.Preconditions;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
@@ -28,10 +26,12 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.tez.service.impl.TezTestService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class MiniTezTestServiceCluster extends AbstractService {
- private static final Log LOG = LogFactory.getLog(MiniTezTestServiceCluster.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MiniTezTestServiceCluster.class);
private final File testWorkDir;
private final long availableMemory;
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
index 379d952..e7c02c8 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
-import org.apache.log4j.Logger;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
import org.apache.tez.common.security.JobTokenIdentifier;
@@ -75,10 +74,12 @@ import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainer
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.TaskSpecProto;
import org.apache.tez.util.ProtoConverters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ContainerRunnerImpl extends AbstractService implements ContainerRunner {
- private static final Logger LOG = Logger.getLogger(ContainerRunnerImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(ContainerRunnerImpl.class);
public static final String DAG_NAME_INSTRUMENTED_FAILURES = "InstrumentedFailures";
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
index 855f1b0..322be00 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
@@ -23,16 +23,16 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
-import org.apache.log4j.Logger;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.service.ContainerRunner;
import org.apache.tez.shufflehandler.ShuffleHandler;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.slf4j.LoggerFactory;
public class TezTestService extends AbstractService implements ContainerRunner {
- private static final Logger LOG = Logger.getLogger(TezTestService.class);
+ private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(TezTestService.class);
private final Configuration shuffleHandlerConf;
private final int numExecutors;
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
index 39d7156..b5d3f83 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
@@ -22,8 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -38,11 +36,13 @@ import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainer
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TezTestServiceProtocolServerImpl extends AbstractService
implements TezTestServiceProtocolBlockingPB {
- private static final Log LOG = LogFactory.getLog(TezTestServiceProtocolServerImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TezTestServiceProtocolServerImpl.class);
private final ContainerRunner containerRunner;
private RPC.Server server;
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java
index 65588fe..294add6 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java
@@ -18,16 +18,16 @@ import java.io.FileDescriptor;
import java.io.IOException;
import java.io.RandomAccessFile;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.ReadaheadPool;
import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.jboss.netty.handler.stream.ChunkedFile;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class FadvisedChunkedFile extends ChunkedFile {
- private static final Log LOG = LogFactory.getLog(FadvisedChunkedFile.class);
+ private static final Logger LOG = LoggerFactory.getLogger(FadvisedChunkedFile.class);
private final boolean manageOsCache;
private final int readaheadLength;
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java
index bdffe52..e5392d3 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java
@@ -22,16 +22,16 @@ import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.ReadaheadPool;
import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.jboss.netty.channel.DefaultFileRegion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class FadvisedFileRegion extends DefaultFileRegion {
- private static final Log LOG = LogFactory.getLog(FadvisedFileRegion.class);
+ private static final Logger LOG = LoggerFactory.getLogger(FadvisedFileRegion.class);
private final boolean manageOsCache;
private final int readaheadLength;
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java
index 9a51ca0..5a45917 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java
@@ -22,20 +22,20 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.tez.runtime.library.common.Constants;
import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
import org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
class IndexCache {
private final Configuration conf;
private final int totalMemoryAllowed;
private AtomicInteger totalMemoryUsed = new AtomicInteger();
- private static final Log LOG = LogFactory.getLog(IndexCache.class);
+ private static final Logger LOG = LoggerFactory.getLogger(IndexCache.class);
private final ConcurrentHashMap<String,IndexInformation> cache =
new ConcurrentHashMap<String,IndexInformation>();
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java
index cc82d74..8cbb8c7 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java
@@ -51,8 +51,6 @@ import java.util.regex.Pattern;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
@@ -105,10 +103,12 @@ import org.jboss.netty.handler.codec.http.QueryStringDecoder;
import org.jboss.netty.handler.ssl.SslHandler;
import org.jboss.netty.handler.stream.ChunkedWriteHandler;
import org.jboss.netty.util.CharsetUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ShuffleHandler {
- private static final Log LOG = LogFactory.getLog(ShuffleHandler.class);
+ private static final Logger LOG = LoggerFactory.getLogger(ShuffleHandler.class);
public static final String SHUFFLE_HANDLER_LOCAL_DIRS = "tez.shuffle.handler.local-dirs";
http://git-wip-us.apache.org/repos/asf/tez/blob/b828d156/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
index b6a166d..4d0a610 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
@@ -20,8 +20,6 @@ import java.io.IOException;
import java.util.Map;
import com.google.common.collect.Maps;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -48,10 +46,12 @@ import org.apache.tez.test.MiniTezCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestExternalTezServices {
- private static final Log LOG = LogFactory.getLog(TestExternalTezServices.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestExternalTezServices.class);
private static final String EXT_PUSH_ENTITY_NAME = "ExtServiceTestPush";
[28/50] [abbrv] tez git commit: TEZ-2357. Tez UI: misc.js.orig is
committed by accident (pramachandran)
Posted by ss...@apache.org.
TEZ-2357. Tez UI: misc.js.orig is committed by accident (pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/67609af4
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/67609af4
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/67609af4
Branch: refs/heads/TEZ-2003
Commit: 67609af4b310c67305cc5e5f87530756ce15a206
Parents: fe11c5e
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Thu Apr 23 19:27:22 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Thu Apr 23 19:27:22 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../webapp/app/scripts/helpers/misc.js.orig | 436 -------------------
2 files changed, 1 insertion(+), 436 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/67609af4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af39092..1c41714 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2357. Tez UI: misc.js.orig is committed by accident
TEZ-2261. Should add diagnostics in DAGAppMaster when recovery error happens
TEZ-2340. TestRecoveryParser fails
TEZ-2345. Tez UI: Enable cell level loading in all DAGs table
http://git-wip-us.apache.org/repos/asf/tez/blob/67609af4/tez-ui/src/main/webapp/app/scripts/helpers/misc.js.orig
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js.orig b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js.orig
deleted file mode 100644
index fdd69bd..0000000
--- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js.orig
+++ /dev/null
@@ -1,436 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-App.Helpers.misc = {
- getStatusClassForEntity: function(status, hasFailedTasks) {
- if(!status) return '';
-
- switch(status) {
- case 'FAILED':
- return 'failed';
- case 'KILLED':
- return 'killed';
- case 'RUNNING':
- return 'running';
- case 'ERROR':
- return 'error';
- case 'SUCCEEDED':
- if (!!hasFailedTasks) {
- return 'warning';
- }
- /*
- TODO: TEZ-2113
- var counterGroups = dag.get('counterGroups');
- var numFailedTasks = this.getCounterValueForDag(counterGroups,
- dag.get('id'), 'org.apache.tez.common.counters.DAGCounter',
- 'NUM_FAILED_TASKS'
- );
-
- if (numFailedTasks > 0) {
- return 'warning';
- }*/
-
- return 'success';
- case 'UNDEFINED':
- return 'unknown';
- default:
- return 'submitted';
- }
- },
-
- getRealStatus: function(entityState, yarnAppState, yarnAppFinalState) {
- if (entityState != 'RUNNING' || (yarnAppState != 'FINISHED' && yarnAppState != 'KILLED' && yarnAppState != 'FAILED')) {
- return entityState;
- }
-
- if (yarnAppState == 'KILLED' || yarnAppState == 'FAILED') {
- return yarnAppState;
- }
-
- return yarnAppFinalState;
- },
-
- getCounterValueForDag: function(counterGroups, dagID, counterGroupName, counterName) {
- if (!counterGroups) {
- return 0;
- }
-
- var cgName = dagID + '/' + counterGroupName;
- var cg = counterGroups.findBy('id', cgName);
- if (!cg) {
- return 0;
- }
- var counters = cg.get('counters');
- if (!counters) {
- return 0;
- }
-
- var counter = counters.findBy('id', cgName + '/' + counterName);
- if (!counter) return 0;
-
- return counter.get('value');
- },
-
- isValidDagStatus: function(status) {
- return $.inArray(status, ['SUBMITTED', 'INITING', 'RUNNING', 'SUCCEEDED',
- 'KILLED', 'FAILED', 'ERROR']) != -1;
- },
-
- isValidTaskStatus: function(status) {
- return $.inArray(status, ['RUNNING', 'SUCCEEDED', 'FAILED', 'KILLED']) != -1;
- },
-
- isStatusInUnsuccessful: function(status) {
- return $.inArray(status, ['FAILED', 'KILLED', 'UNDEFINED']) != -1;
- },
-
- /**
- * To trim a complete class path with namespace to the class name.
- */
- getClassName: function (classPath) {
- return classPath.substr(classPath.lastIndexOf('.') + 1);
- },
-
- /*
- * Normalizes counter style configurations
- * @param counterConfigs Array
- * @return Normalized configurations
- */
- normalizeCounterConfigs: function (counterConfigs) {
- return counterConfigs.map(function (configuration) {
- configuration.headerCellName = configuration.counterName || configuration.counterId;
- configuration.id = '%@/%@'.fmt(configuration.counterGroupName || configuration.groupId,
- configuration.counterName || configuration.counterId),
- configuration.getCellContent = App.Helpers.misc.getCounterCellContent;
- return configuration;
- });
- },
-
- /*
- * Creates column definitions form configuration object array
- * @param columnConfigs Array
- * @return columnDefinitions Array
- */
- createColumnsFromConfigs: function (columnConfigs) {
- return columnConfigs.map(function (columnConfig) {
- if(columnConfig.getCellContentHelper) {
- columnConfig.getCellContent = App.Helpers.get(columnConfig.getCellContentHelper);
- }
- columnConfig.minWidth = columnConfig.minWidth || 135;
-
- return columnConfig.filterID ?
- App.ExTable.ColumnDefinition.createWithMixins(App.ExTable.FilterColumnMixin, columnConfig) :
- App.ExTable.ColumnDefinition.create(columnConfig);
- });
- },
-
- /*
- * Returns a counter value from for a row
- * @param row
- * @return value
- */
- getCounterCellContent: function (row) {
- var contentPath = this.id.split('/'),
- group = contentPath[0],
- counter = contentPath[1],
- id = row.get('id'),
- value = 'Not Available';
-
- try{
- value = row.get('counterGroups').
- findBy('id', '%@/%@'.fmt(id, group)).
- get('counters').
- findBy('id', '%@/%@/%@'.fmt(id, group, counter)).
- get('value');
- }catch(e){}
-
- return App.Helpers.number.formatNumThousands(value);
- },
-
- /*
- * returns a formatted message, the real cause is unknown and the error object details
- * depends on the error cause. the function tries to handle ajax error or a native errors
- */
- formatError: function(error, defaultErrorMessage) {
- var msg;
- // for cross domain requests, the error is not set if no access control headers were found.
- // this could be either because there was a n/w error or the cors headers being not set.
- if (error.status === 0 && error.statusText === 'error') {
- msg = defaultErrorMessage ;
- } else {
- msg = error.statusText || error.message;
- }
- msg = msg || 'Unknown error';
- if (!!error.responseText) {
- msg += error.responseText;
- }
-
- if(error.requestOptions) {
- msg = '%@<br/>Could not retrieve expected data from %@ @ %@'.fmt(
- msg,
- error.requestOptions.targetServer,
- error.requestOptions.url
- )
- }
-
- return {
- errCode: error.status || 'Unknown',
- msg: msg,
- details: error.stack
- };
- },
-
- /**
- * Normalize path
- * @param path {String}
- * @return normalized path {String}
- */
- normalizePath: function (path) {
- if(path && path.charAt(path.length - 1) == '/') {
- path = path.slice(0, -1);
- }
- return path;
- },
-
- // Tez originally shows the status for task and task attempt only on
- // completion. this causes confusion to the user as the status would not be
- // displayed. so if status is not set return the status as 'RUNNING'. We do
- // not diffentiate between running and scheduled.
- getFixedupDisplayStatus: function(originalStatus) {
- // if status is not set show it as running, since originally the task did
- // not have a status set on scheduled/running.
- // with the new version we set the status of task as scheduled and that of
- // task attempt as running
- if (!originalStatus || originalStatus == 'SCHEDULED') {
- originalStatus = 'RUNNING';
- }
- return originalStatus;
- },
-
- /**
- * Merge content of obj2 into obj2, array elements will be concated.
- * @param obj1 {Object}
- * @param obj2 {Object}
- */
- merge: function objectMerge(obj1, obj2) {
- $.each(obj2, function (key, val) {
- if(Array.isArray(obj1[key]) && Array.isArray(val)) {
- $.merge(obj1[key], val);
- }
- else if($.isPlainObject(obj1[key]) && $.isPlainObject(val)) {
- objectMerge(obj1[key], val);
- }
- else {
- obj1[key] = val;
- }
- });
- },
-
- getTaskIndex: function(dagID, taskID) {
- var idPrefix = 'task_%@_'.fmt(dagID.substr(4));
- return taskID.indexOf(idPrefix) == 0 ? taskID.substr(idPrefix.length) : id;
- },
-
- getVertexIdFromName: function(idToNameMap, vertexName) {
- idToNameMap = idToNameMap || {};
- var vertexId = undefined;
- $.each(idToNameMap, function(id, name) {
- if (name === vertexName) {
- vertexId = id;
- return false;
- }
- });
- return vertexId;
- },
-
- /**
- * Remove the specific record from store
- * @param store {DS.Store}
- * @param type {String}
- * @param id {String}
- */
- removeRecord: function (store, type, id) {
- var record = store.getById(type, id);
- if(record) {
- store.unloadRecord(record);
- }
- },
-
- downloadDAG: function(dagID, options) {
- var opts = options || {},
- batchSize = opts.batchSize || 1000,
- baseurl = '%@/%@'.fmt(App.env.timelineBaseUrl, App.Configs.restNamespace.timeline),
- itemsToDownload = [
- {
- url: getUrl('TEZ_DAG_ID', dagID),
- context: { name: 'dag', type: 'TEZ_DAG_ID' },
- onItemFetched: processSingleItem
- },
- {
- url: getUrl('TEZ_VERTEX_ID', dagID),
- context: { name: 'vertices', type: 'TEZ_VERTEX_ID', part: 0 },
- onItemFetched: processMultipleItems
- },
- {
- url: getUrl('TEZ_TASK_ID', dagID),
- context: { name: 'tasks', type: 'TEZ_TASK_ID', part: 0 },
- onItemFetched: processMultipleItems
- },
- {
- url: getUrl('TEZ_TASK_ATTEMPT_ID', dagID),
- context: { name: 'task_attempts', type: 'TEZ_TASK_ATTEMPT_ID', part: 0 },
- onItemFetched: processMultipleItems
- }
- ],
- numItemTypesToDownload = itemsToDownload.length,
- downloader = App.Helpers.io.fileDownloader(),
- zipHelper = App.Helpers.io.zipHelper({
- onProgress: function(filename, current, total) {
- Em.Logger.debug('%@: %@ of %@'.fmt(filename, current, total));
- },
- onAdd: function(filename) {
- Em.Logger.debug('adding %@ to Zip'.fmt(filename));
- }
- });
-
- function getUrl(type, dagID, fromID) {
- var url;
- if (type == 'TEZ_DAG_ID') {
- url = '%@/%@/%@'.fmt(baseurl, type, dagID);
- } else {
- url = '%@/%@?primaryFilter=TEZ_DAG_ID:%@&limit=%@'.fmt(baseurl, type, dagID, batchSize + 1);
- if (!!fromID) {
- url = '%@&fromId=%@'.fmt(url, fromID);
- }
- }
- return url;
- }
-
- function checkIfAllDownloaded() {
- numItemTypesToDownload--;
- if (numItemTypesToDownload == 0) {
- downloader.finish();
- }
- }
-
- function processSingleItem(data, context) {
- var obj = {};
- obj[context.name] = data;
-
- zipHelper.addFile({name: '%@.json'.fmt(context.name), data: JSON.stringify(obj, null, 2)});
- checkIfAllDownloaded();
- }
-
- function processMultipleItems(data, context) {
- var obj = {};
- var nextBatchStart = undefined;
-
- if (!$.isArray(data.entities)) {
- throw "invalid data";
- }
-
- // need to handle no more entries , zero entries
- if (data.entities.length > batchSize) {
- nextBatchStart = data.entities.pop().entity;
- }
- obj[context.name] = data.entities;
-
- zipHelper.addFile({name: '%@_part_%@.json'.fmt(context.name, context.part), data: JSON.stringify(obj, null, 2)});
-
- if (!!nextBatchStart) {
- context.part++;
- downloader.queueItem({
- url: getUrl(context.type, dagID, nextBatchStart),
- context: context,
- onItemFetched: processMultipleItems
- });
- } else {
- checkIfAllDownloaded();
- }
- }
-
- downloader.queueItems(itemsToDownload);
-
- downloader.then(function() {
- Em.Logger.info('Finished download');
- zipHelper.close();
- }).catch(function() {
- Em.Logger.error('Failed to download');
- zipHelper.abort();
- });
-
- var that = this;
- zipHelper.then(function(zippedBlob) {
- saveAs(zippedBlob, '%@.zip'.fmt(dagID));
- if ($.isFunction(opts.onSuccess)) {
- opts.onSuccess();
- }
- }).catch(function() {
- Em.Logger.error('zip Failed');
- if ($.isFunction(opts.onFailure)) {
- opts.onFailure();
- }
- });
-
- return {
- cancel: function() {
- downloader.cancel();
- }
- }
- },
-
- dagStatusUIOptions: [
- { label: 'All', id: null },
- { label: 'Submitted', id: 'SUBMITTED' },
- { label: 'Running', id: 'RUNNING' },
- { label: 'Succeeded', id: 'SUCCEEDED' },
- { label: 'Failed', id: 'FAILED' },
- { label: 'Killed', id: 'KILLED' },
- { label: 'Error', id: 'ERROR' },
- ],
-
- vertexStatusUIOptions: [
- { label: 'All', id: null },
- { label: 'Running', id: 'RUNNING' },
- { label: 'Succeeded', id: 'SUCCEEDED' },
- { label: 'Failed', id: 'FAILED' },
- { label: 'Killed', id: 'KILLED' },
- { label: 'Error', id: 'ERROR' },
- ],
-
- taskStatusUIOptions: [
- { label: 'All', id: null },
- { label: 'Running', id: 'SCHEDULED' },
- { label: 'Succeeded', id: 'SUCCEEDED' },
- { label: 'Failed', id: 'FAILED' },
- { label: 'Killed', id: 'KILLED' },
- ],
-
- taskAttemptStatusUIOptions: [
- { label: 'All', id: null },
- { label: 'Running', id: 'RUNNING' },
- { label: 'Succeeded', id: 'SUCCEEDED' },
- { label: 'Failed', id: 'FAILED' },
- { label: 'Killed', id: 'KILLED' },
- ],
-
- defaultQueryParamsConfig: {
- refreshModel: true,
- replace: true
- }
-
-}
[33/50] [abbrv] tez git commit: TEZ-2090. Add tests for jobs running
in external services. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
new file mode 100644
index 0000000..4a6ce33
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
@@ -0,0 +1,512 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
+import org.apache.log4j.Logger;
+import org.apache.tez.common.TezCommonUtils;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.runtime.task.TaskReporter;
+import org.apache.tez.runtime.task.TezTaskRunner;
+import org.apache.tez.service.ContainerRunner;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.runtime.api.ExecutionContext;
+import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
+import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl;
+import org.apache.tez.runtime.task.TezChild;
+import org.apache.tez.runtime.task.TezChild.ContainerExecutionResult;
+import org.apache.tez.shufflehandler.ShuffleHandler;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
+import org.apache.tez.util.ProtoConverters;
+
+public class ContainerRunnerImpl extends AbstractService implements ContainerRunner {
+
+ private static final Logger LOG = Logger.getLogger(ContainerRunnerImpl.class);
+
+ private final ListeningExecutorService executorService;
+ private final AtomicReference<InetSocketAddress> localAddress;
+ private final String[] localDirsBase;
+ private final Map<String, String> localEnv = new HashMap<String, String>();
+ private volatile FileSystem localFs;
+ private final long memoryPerExecutor;
+ // TODO Support for removing queued containers, interrupting / killing specific containers - when preemption is supported
+
+
+
+
+ public ContainerRunnerImpl(int numExecutors, String[] localDirsBase,
+ AtomicReference<InetSocketAddress> localAddress,
+ long totalMemoryAvailableBytes) {
+ super("ContainerRunnerImpl");
+ Preconditions.checkState(numExecutors > 0,
+ "Invalid number of executors: " + numExecutors + ". Must be > 0");
+ this.localDirsBase = localDirsBase;
+ this.localAddress = localAddress;
+
+ ExecutorService raw = Executors.newFixedThreadPool(numExecutors,
+ new ThreadFactoryBuilder().setNameFormat("ContainerExecutor %d").build());
+ this.executorService = MoreExecutors.listeningDecorator(raw);
+
+
+ // 80% of memory considered for accounted buffers. Rest for objects.
+ // TODO Tune this based on the available size.
+ this.memoryPerExecutor = (long)(totalMemoryAvailableBytes * 0.8 / (float) numExecutors);
+
+ LOG.info("ContainerRunnerImpl config: " +
+ "memoryPerExecutorDerived=" + memoryPerExecutor +
+ ", numExecutors=" + numExecutors
+ );
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ try {
+ localFs = FileSystem.getLocal(conf);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to setup local filesystem instance", e);
+ }
+ }
+
+ @Override
+ public void serviceStart() {
+ }
+
+ public void setShufflePort(int shufflePort) {
+ AuxiliaryServiceHelper.setServiceDataIntoEnv(
+ TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
+ ByteBuffer.allocate(4).putInt(shufflePort), localEnv);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ }
+
+ // TODO Move this into a utilities class
+ private static String createAppSpecificLocalDir(String baseDir, String applicationIdString,
+ String user) {
+ return baseDir + File.separator + "usercache" + File.separator + user + File.separator +
+ "appcache" + File.separator + applicationIdString;
+ }
+
+ /**
+ * Submit a container which is ready for running.
+ * The regular pull mechanism will be used to fetch work from the AM
+ * @param request
+ * @throws IOException
+ */
+ @Override
+ public void queueContainer(RunContainerRequestProto request) throws IOException {
+ LOG.info("Queuing container for execution: " + request);
+
+ Map<String, String> env = new HashMap<String, String>();
+ env.putAll(localEnv);
+ env.put(ApplicationConstants.Environment.USER.name(), request.getUser());
+
+ String[] localDirs = new String[localDirsBase.length];
+
+ // Setup up local dirs to be application specific, and create them.
+ for (int i = 0; i < localDirsBase.length; i++) {
+ localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(),
+ request.getUser());
+ localFs.mkdirs(new Path(localDirs[i]));
+ }
+ LOG.info("DEBUG: Dirs are: " + Arrays.toString(localDirs));
+
+
+ // Setup workingDir. This is otherwise setup as Environment.PWD
+ // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream)
+ String workingDir = localDirs[0];
+
+ Credentials credentials = new Credentials();
+ DataInputBuffer dib = new DataInputBuffer();
+ byte[] tokenBytes = request.getCredentialsBinary().toByteArray();
+ dib.reset(tokenBytes, tokenBytes.length);
+ credentials.readTokenStorageStream(dib);
+
+ Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
+
+ // TODO Unregistering does not happen at the moment, since there's no signals on when an app completes.
+ LOG.info("DEBUG: Registering request with the ShuffleHandler");
+ ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser());
+
+
+ ContainerRunnerCallable callable = new ContainerRunnerCallable(request, new Configuration(getConfig()),
+ new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs,
+ workingDir, credentials, memoryPerExecutor);
+ ListenableFuture<ContainerExecutionResult> future = executorService
+ .submit(callable);
+ Futures.addCallback(future, new ContainerRunnerCallback(request, callable));
+ }
+
+ /**
+ * Submit an entire work unit - containerId + TaskSpec.
+ * This is intended for a task push from the AM
+ *
+ * @param request
+ * @throws IOException
+ */
+ @Override
+ public void submitWork(SubmitWorkRequestProto request) throws
+ IOException {
+ LOG.info("Queuing work for execution: " + request);
+
+ Map<String, String> env = new HashMap<String, String>();
+ env.putAll(localEnv);
+ env.put(ApplicationConstants.Environment.USER.name(), request.getUser());
+
+ String[] localDirs = new String[localDirsBase.length];
+
+ // Setup up local dirs to be application specific, and create them.
+ for (int i = 0; i < localDirsBase.length; i++) {
+ localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(),
+ request.getUser());
+ localFs.mkdirs(new Path(localDirs[i]));
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Dirs are: " + Arrays.toString(localDirs));
+ }
+
+ // Setup workingDir. This is otherwise setup as Environment.PWD
+ // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream)
+ String workingDir = localDirs[0];
+
+ Credentials credentials = new Credentials();
+ DataInputBuffer dib = new DataInputBuffer();
+ byte[] tokenBytes = request.getCredentialsBinary().toByteArray();
+ dib.reset(tokenBytes, tokenBytes.length);
+ credentials.readTokenStorageStream(dib);
+
+ Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
+
+ // TODO Unregistering does not happen at the moment, since there's no signals on when an app completes.
+ LOG.info("DEBUG: Registering request with the ShuffleHandler");
+ ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser());
+ TaskRunnerCallable callable = new TaskRunnerCallable(request, new Configuration(getConfig()),
+ new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs,
+ workingDir, credentials, memoryPerExecutor);
+ ListenableFuture<ContainerExecutionResult> future = executorService.submit(callable);
+ Futures.addCallback(future, new TaskRunnerCallback(request, callable));
+ }
+
+
+ static class ContainerRunnerCallable implements Callable<ContainerExecutionResult> {
+
+ private final RunContainerRequestProto request;
+ private final Configuration conf;
+ private final String workingDir;
+ private final String[] localDirs;
+ private final Map<String, String> envMap;
+ private final String pid = null;
+ private final ObjectRegistryImpl objectRegistry;
+ private final ExecutionContext executionContext;
+ private final Credentials credentials;
+ private final long memoryAvailable;
+ private volatile TezChild tezChild;
+
+
+ ContainerRunnerCallable(RunContainerRequestProto request, Configuration conf,
+ ExecutionContext executionContext, Map<String, String> envMap,
+ String[] localDirs, String workingDir, Credentials credentials,
+ long memoryAvailable) {
+ this.request = request;
+ this.conf = conf;
+ this.executionContext = executionContext;
+ this.envMap = envMap;
+ this.workingDir = workingDir;
+ this.localDirs = localDirs;
+ this.objectRegistry = new ObjectRegistryImpl();
+ this.credentials = credentials;
+ this.memoryAvailable = memoryAvailable;
+
+ }
+
+ @Override
+ public ContainerExecutionResult call() throws Exception {
+ Stopwatch sw = new Stopwatch().start();
+ tezChild =
+ new TezChild(conf, request.getAmHost(), request.getAmPort(),
+ request.getContainerIdString(),
+ request.getTokenIdentifier(), request.getAppAttemptNumber(), workingDir, localDirs,
+ envMap, objectRegistry, pid,
+ executionContext, credentials, memoryAvailable, request.getUser());
+ ContainerExecutionResult result = tezChild.run();
+ LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" +
+ sw.stop().elapsedMillis());
+ return result;
+ }
+
+ public TezChild getTezChild() {
+ return this.tezChild;
+ }
+ }
+
+
+ final class ContainerRunnerCallback implements FutureCallback<ContainerExecutionResult> {
+
+ private final RunContainerRequestProto request;
+ private final ContainerRunnerCallable containerRunnerCallable;
+
+ ContainerRunnerCallback(RunContainerRequestProto request,
+ ContainerRunnerCallable containerRunnerCallable) {
+ this.request = request;
+ this.containerRunnerCallable = containerRunnerCallable;
+ }
+
+ // TODO Proper error handling
+ @Override
+ public void onSuccess(ContainerExecutionResult result) {
+ switch (result.getExitStatus()) {
+ case SUCCESS:
+ LOG.info("Successfully finished: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString());
+ break;
+ case EXECUTION_FAILURE:
+ LOG.info("Failed to run: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), result.getThrowable());
+ break;
+ case INTERRUPTED:
+ LOG.info(
+ "Interrupted while running: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), result.getThrowable());
+ break;
+ case ASKED_TO_DIE:
+ LOG.info(
+ "Asked to die while running: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString());
+ break;
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.error(
+ "TezChild execution failed for : " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), t);
+ TezChild tezChild = containerRunnerCallable.getTezChild();
+ if (tezChild != null) {
+ tezChild.shutdown();
+ }
+ }
+ }
+
+ static class TaskRunnerCallable implements Callable<ContainerExecutionResult> {
+
+ private final SubmitWorkRequestProto request;
+ private final Configuration conf;
+ private final String workingDir;
+ private final String[] localDirs;
+ private final Map<String, String> envMap;
+ private final String pid = null;
+ private final ObjectRegistryImpl objectRegistry;
+ private final ExecutionContext executionContext;
+ private final Credentials credentials;
+ private final long memoryAvailable;
+ private final ListeningExecutorService executor;
+ private volatile TezTaskRunner taskRunner;
+ private volatile TaskReporter taskReporter;
+ private TezTaskUmbilicalProtocol umbilical;
+
+
+ TaskRunnerCallable(SubmitWorkRequestProto request, Configuration conf,
+ ExecutionContext executionContext, Map<String, String> envMap,
+ String[] localDirs, String workingDir, Credentials credentials,
+ long memoryAvailable) {
+ this.request = request;
+ this.conf = conf;
+ this.executionContext = executionContext;
+ this.envMap = envMap;
+ this.workingDir = workingDir;
+ this.localDirs = localDirs;
+ this.objectRegistry = new ObjectRegistryImpl();
+ this.credentials = credentials;
+ this.memoryAvailable = memoryAvailable;
+ // TODO This executor seems unnecessary. Here and TezChild
+ ExecutorService executorReal = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat("TezTaskRunner_" + request.getTaskSpec().getTaskAttemptIdString()).build());
+ executor = MoreExecutors.listeningDecorator(executorReal);
+ }
+
+ @Override
+ public ContainerExecutionResult call() throws Exception {
+
+ // TODO Consolidate this code with TezChild.
+ Stopwatch sw = new Stopwatch().start();
+ UserGroupInformation taskUgi = UserGroupInformation.createRemoteUser(request.getUser());
+ taskUgi.addCredentials(credentials);
+
+ Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
+ Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>();
+ serviceConsumerMetadata.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
+ TezCommonUtils.convertJobTokenToBytes(jobToken));
+ Multimap<String, String> startedInputsMap = HashMultimap.create();
+
+ UserGroupInformation taskOwner =
+ UserGroupInformation.createRemoteUser(request.getTokenIdentifier());
+ final InetSocketAddress address =
+ NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort());
+ SecurityUtil.setTokenService(jobToken, address);
+ taskOwner.addToken(jobToken);
+ umbilical = taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() {
+ @Override
+ public TezTaskUmbilicalProtocol run() throws Exception {
+ return RPC.getProxy(TezTaskUmbilicalProtocol.class,
+ TezTaskUmbilicalProtocol.versionID, address, conf);
+ }
+ });
+ // TODO Stop reading this on each request.
+ taskReporter = new TaskReporter(
+ umbilical,
+ conf.getInt(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS,
+ TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT),
+ conf.getLong(
+ TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS,
+ TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS_DEFAULT),
+ conf.getInt(TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT,
+ TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT),
+ new AtomicLong(0),
+ request.getContainerIdString());
+
+ taskRunner = new TezTaskRunner(conf, taskUgi, localDirs,
+ ProtoConverters.getTaskSpecfromProto(request.getTaskSpec()), umbilical,
+ request.getAppAttemptNumber(),
+ serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor, objectRegistry,
+ pid,
+ executionContext, memoryAvailable);
+
+ boolean shouldDie;
+ try {
+ shouldDie = !taskRunner.run();
+ if (shouldDie) {
+ LOG.info("Got a shouldDie notification via hearbeats. Shutting down");
+ return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null,
+ "Asked to die by the AM");
+ }
+ } catch (IOException e) {
+ return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
+ e, "TaskExecutionFailure: " + e.getMessage());
+ } catch (TezException e) {
+ return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
+ e, "TaskExecutionFailure: " + e.getMessage());
+ } finally {
+ FileSystem.closeAllForUGI(taskUgi);
+ }
+ LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" +
+ sw.stop().elapsedMillis());
+ return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null,
+ null);
+ }
+
+ public void shutdown() {
+ executor.shutdownNow();
+ if (taskReporter != null) {
+ taskReporter.shutdown();
+ }
+ if (umbilical != null) {
+ RPC.stopProxy(umbilical);
+ }
+ }
+ }
+
+
+ final class TaskRunnerCallback implements FutureCallback<ContainerExecutionResult> {
+
+ private final SubmitWorkRequestProto request;
+ private final TaskRunnerCallable taskRunnerCallable;
+
+ TaskRunnerCallback(SubmitWorkRequestProto request,
+ TaskRunnerCallable containerRunnerCallable) {
+ this.request = request;
+ this.taskRunnerCallable = containerRunnerCallable;
+ }
+
+ // TODO Proper error handling
+ @Override
+ public void onSuccess(ContainerExecutionResult result) {
+ switch (result.getExitStatus()) {
+ case SUCCESS:
+ LOG.info("Successfully finished: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString());
+ break;
+ case EXECUTION_FAILURE:
+ LOG.info("Failed to run: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), result.getThrowable());
+ break;
+ case INTERRUPTED:
+ LOG.info(
+ "Interrupted while running: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), result.getThrowable());
+ break;
+ case ASKED_TO_DIE:
+ LOG.info(
+ "Asked to die while running: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString());
+ break;
+ }
+ taskRunnerCallable.shutdown();
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.error(
+ "TezTaskRunner execution failed for : " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), t);
+ taskRunnerCallable.shutdown();
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
new file mode 100644
index 0000000..012e352
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.tez.service.ContainerRunner;
+import org.apache.tez.shufflehandler.ShuffleHandler;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+
+public class TezTestService extends AbstractService implements ContainerRunner {
+
+ private static final Logger LOG = Logger.getLogger(TezTestService.class);
+
+ private final Configuration shuffleHandlerConf;
+ private final int numExecutors;
+
+ private final TezTestServiceProtocolServerImpl server;
+ private final ContainerRunnerImpl containerRunner;
+ private final String[] localDirs;
+
+ private final AtomicInteger numSubmissions = new AtomicInteger(0);
+
+
+ private final AtomicReference<InetSocketAddress> address = new AtomicReference<InetSocketAddress>();
+
+ public TezTestService(Configuration conf, int numExecutors, long memoryAvailable, String[] localDirs) {
+ super(TezTestService.class.getSimpleName());
+ this.numExecutors = numExecutors;
+ this.localDirs = localDirs;
+
+ long memoryAvailableBytes = memoryAvailable;
+ long jvmMax = Runtime.getRuntime().maxMemory();
+
+ LOG.info(TezTestService.class.getSimpleName() + " created with the following configuration: " +
+ "numExecutors=" + numExecutors +
+ ", workDirs=" + Arrays.toString(localDirs) +
+ ", memoryAvailable=" + memoryAvailable +
+ ", jvmMaxMemory=" + jvmMax);
+
+ Preconditions.checkArgument(this.numExecutors > 0);
+ Preconditions.checkArgument(this.localDirs != null && this.localDirs.length > 0,
+ "Work dirs must be specified");
+ Preconditions.checkState(jvmMax >= memoryAvailableBytes,
+ "Invalid configuration. Xmx value too small. maxAvailable=" + jvmMax + ", configured=" +
+ memoryAvailableBytes);
+
+ this.shuffleHandlerConf = new Configuration(conf);
+ // Start Shuffle on a random port
+ this.shuffleHandlerConf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+ this.shuffleHandlerConf.set(ShuffleHandler.SHUFFLE_HANDLER_LOCAL_DIRS, StringUtils.arrayToString(localDirs));
+
+ this.server = new TezTestServiceProtocolServerImpl(this, address);
+ this.containerRunner = new ContainerRunnerImpl(numExecutors, localDirs, address,
+ memoryAvailableBytes);
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ server.init(conf);
+ containerRunner.init(conf);
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ ShuffleHandler.initializeAndStart(shuffleHandlerConf);
+ containerRunner.setShufflePort(ShuffleHandler.get().getPort());
+ server.start();
+ containerRunner.start();
+ }
+
+ public void serviceStop() throws Exception {
+ containerRunner.stop();
+ server.stop();
+ ShuffleHandler.get().stop();
+ }
+
+ public InetSocketAddress getListenerAddress() {
+ return server.getBindAddress();
+ }
+
+ public int getShufflePort() {
+ return ShuffleHandler.get().getPort();
+ }
+
+
+
+ @Override
+ public void queueContainer(RunContainerRequestProto request) throws IOException {
+ numSubmissions.incrementAndGet();
+ containerRunner.queueContainer(request);
+ }
+
+ @Override
+ public void submitWork(TezTestServiceProtocolProtos.SubmitWorkRequestProto request) throws
+ IOException {
+ numSubmissions.incrementAndGet();
+ containerRunner.submitWork(request);
+ }
+
+ public int getNumSubmissions() {
+ return numSubmissions.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java
new file mode 100644
index 0000000..10d2952
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.tez.service.TezTestServiceProtocolBlockingPB;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto;
+
+
+public class TezTestServiceProtocolClientImpl implements TezTestServiceProtocolBlockingPB {
+
+ private final Configuration conf;
+ private final InetSocketAddress serverAddr;
+ TezTestServiceProtocolBlockingPB proxy;
+
+
+ public TezTestServiceProtocolClientImpl(Configuration conf, String hostname, int port) {
+ this.conf = conf;
+ this.serverAddr = NetUtils.createSocketAddr(hostname, port);
+ }
+
+ @Override
+ public RunContainerResponseProto runContainer(RpcController controller,
+ RunContainerRequestProto request) throws
+ ServiceException {
+ try {
+ return getProxy().runContainer(null, request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public TezTestServiceProtocolProtos.SubmitWorkResponseProto submitWork(RpcController controller,
+ TezTestServiceProtocolProtos.SubmitWorkRequestProto request) throws
+ ServiceException {
+ try {
+ return getProxy().submitWork(null, request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+
+ public TezTestServiceProtocolBlockingPB getProxy() throws IOException {
+ if (proxy == null) {
+ proxy = createProxy();
+ }
+ return proxy;
+ }
+
+ public TezTestServiceProtocolBlockingPB createProxy() throws IOException {
+ TezTestServiceProtocolBlockingPB p;
+ // TODO Fix security
+ RPC.setProtocolEngine(conf, TezTestServiceProtocolBlockingPB.class, ProtobufRpcEngine.class);
+ p = (TezTestServiceProtocolBlockingPB) RPC
+ .getProxy(TezTestServiceProtocolBlockingPB.class, 0, serverAddr, conf);
+ return p;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
new file mode 100644
index 0000000..d7f8444
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.tez.service.ContainerRunner;
+import org.apache.tez.service.TezTestServiceProtocolBlockingPB;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto;
+
+public class TezTestServiceProtocolServerImpl extends AbstractService
+ implements TezTestServiceProtocolBlockingPB {
+
+ private static final Log LOG = LogFactory.getLog(TezTestServiceProtocolServerImpl.class);
+
+ private final ContainerRunner containerRunner;
+ private RPC.Server server;
+ private final AtomicReference<InetSocketAddress> bindAddress;
+
+
+ public TezTestServiceProtocolServerImpl(ContainerRunner containerRunner,
+ AtomicReference<InetSocketAddress> address) {
+ super(TezTestServiceProtocolServerImpl.class.getSimpleName());
+ this.containerRunner = containerRunner;
+ this.bindAddress = address;
+ }
+
+ @Override
+ public RunContainerResponseProto runContainer(RpcController controller,
+ RunContainerRequestProto request) throws
+ ServiceException {
+ LOG.info("Received request: " + request);
+ try {
+ containerRunner.queueContainer(request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ return RunContainerResponseProto.getDefaultInstance();
+ }
+
+ @Override
+ public SubmitWorkResponseProto submitWork(RpcController controller, TezTestServiceProtocolProtos.SubmitWorkRequestProto request) throws
+ ServiceException {
+ LOG.info("Received submitWork request: " + request);
+ try {
+ containerRunner.submitWork(request);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return SubmitWorkResponseProto.getDefaultInstance();
+ }
+
+
+ @Override
+ public void serviceStart() {
+ Configuration conf = getConfig();
+
+ int numHandlers = 3;
+ InetSocketAddress addr = new InetSocketAddress(0);
+
+ try {
+ server = createServer(TezTestServiceProtocolBlockingPB.class, addr, conf, numHandlers,
+ TezTestServiceProtocolProtos.TezTestServiceProtocol.newReflectiveBlockingService(this));
+ server.start();
+ } catch (IOException e) {
+ LOG.error("Failed to run RPC Server", e);
+ throw new RuntimeException(e);
+ }
+
+ InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server);
+ this.bindAddress.set(NetUtils.createSocketAddrForHost(
+ serverBindAddress.getAddress().getCanonicalHostName(),
+ serverBindAddress.getPort()));
+ LOG.info("Instantiated TestTestServiceListener at " + bindAddress);
+ }
+
+ @Override
+ public void serviceStop() {
+ if (server != null) {
+ server.stop();
+ }
+ }
+
+ @InterfaceAudience.Private
+ @VisibleForTesting
+ InetSocketAddress getBindAddress() {
+ return this.bindAddress.get();
+ }
+
+ private RPC.Server createServer(Class<?> pbProtocol, InetSocketAddress addr, Configuration conf,
+ int numHandlers, BlockingService blockingService) throws
+ IOException {
+ RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class);
+ RPC.Server server = new RPC.Builder(conf)
+ .setProtocol(pbProtocol)
+ .setInstance(blockingService)
+ .setBindAddress(addr.getHostName())
+ .setPort(0)
+ .setNumHandlers(numHandlers)
+ .build();
+ // TODO Add security.
+ return server;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java
new file mode 100644
index 0000000..65588fe
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.shufflehandler;
+
+import java.io.FileDescriptor;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.ReadaheadPool;
+import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.jboss.netty.handler.stream.ChunkedFile;
+
+public class FadvisedChunkedFile extends ChunkedFile {
+
+ private static final Log LOG = LogFactory.getLog(FadvisedChunkedFile.class);
+
+ private final boolean manageOsCache;
+ private final int readaheadLength;
+ private final ReadaheadPool readaheadPool;
+ private final FileDescriptor fd;
+ private final String identifier;
+
+ private ReadaheadRequest readaheadRequest;
+
+ public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
+ int chunkSize, boolean manageOsCache, int readaheadLength,
+ ReadaheadPool readaheadPool, String identifier) throws IOException {
+ super(file, position, count, chunkSize);
+ this.manageOsCache = manageOsCache;
+ this.readaheadLength = readaheadLength;
+ this.readaheadPool = readaheadPool;
+ this.fd = file.getFD();
+ this.identifier = identifier;
+ }
+
+ @Override
+ public Object nextChunk() throws Exception {
+ if (manageOsCache && readaheadPool != null) {
+ readaheadRequest = readaheadPool
+ .readaheadStream(identifier, fd, getCurrentOffset(), readaheadLength,
+ getEndOffset(), readaheadRequest);
+ }
+ return super.nextChunk();
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (readaheadRequest != null) {
+ readaheadRequest.cancel();
+ }
+ if (manageOsCache && getEndOffset() - getStartOffset() > 0) {
+ try {
+ NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
+ fd,
+ getStartOffset(), getEndOffset() - getStartOffset(),
+ NativeIO.POSIX.POSIX_FADV_DONTNEED);
+ } catch (Throwable t) {
+ LOG.warn("Failed to manage OS cache for " + identifier, t);
+ }
+ }
+ super.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java
new file mode 100644
index 0000000..bdffe52
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.shufflehandler;
+
+import java.io.FileDescriptor;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.ReadaheadPool;
+import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.jboss.netty.channel.DefaultFileRegion;
+
+public class FadvisedFileRegion extends DefaultFileRegion {
+
+ private static final Log LOG = LogFactory.getLog(FadvisedFileRegion.class);
+
+ private final boolean manageOsCache;
+ private final int readaheadLength;
+ private final ReadaheadPool readaheadPool;
+ private final FileDescriptor fd;
+ private final String identifier;
+ private final long count;
+ private final long position;
+ private final int shuffleBufferSize;
+ private final boolean shuffleTransferToAllowed;
+ private final FileChannel fileChannel;
+
+ private ReadaheadRequest readaheadRequest;
+
+ public FadvisedFileRegion(RandomAccessFile file, long position, long count,
+ boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
+ String identifier, int shuffleBufferSize,
+ boolean shuffleTransferToAllowed) throws IOException {
+ super(file.getChannel(), position, count);
+ this.manageOsCache = manageOsCache;
+ this.readaheadLength = readaheadLength;
+ this.readaheadPool = readaheadPool;
+ this.fd = file.getFD();
+ this.identifier = identifier;
+ this.fileChannel = file.getChannel();
+ this.count = count;
+ this.position = position;
+ this.shuffleBufferSize = shuffleBufferSize;
+ this.shuffleTransferToAllowed = shuffleTransferToAllowed;
+ }
+
+ @Override
+ public long transferTo(WritableByteChannel target, long position)
+ throws IOException {
+ if (manageOsCache && readaheadPool != null) {
+ readaheadRequest = readaheadPool.readaheadStream(identifier, fd,
+ getPosition() + position, readaheadLength,
+ getPosition() + getCount(), readaheadRequest);
+ }
+
+ if(this.shuffleTransferToAllowed) {
+ return super.transferTo(target, position);
+ } else {
+ return customShuffleTransfer(target, position);
+ }
+ }
+
+ /**
+ * This method transfers data using local buffer. It transfers data from
+ * a disk to a local buffer in memory, and then it transfers data from the
+ * buffer to the target. This is used only if transferTo is disallowed in
+ * the configuration file. super.TransferTo does not perform well on Windows
+ * due to a small IO request generated. customShuffleTransfer can control
+ * the size of the IO requests by changing the size of the intermediate
+ * buffer.
+ */
+ @VisibleForTesting
+ long customShuffleTransfer(WritableByteChannel target, long position)
+ throws IOException {
+ long actualCount = this.count - position;
+ if (actualCount < 0 || position < 0) {
+ throw new IllegalArgumentException(
+ "position out of range: " + position +
+ " (expected: 0 - " + (this.count - 1) + ')');
+ }
+ if (actualCount == 0) {
+ return 0L;
+ }
+
+ long trans = actualCount;
+ int readSize;
+ ByteBuffer byteBuffer = ByteBuffer.allocate(this.shuffleBufferSize);
+
+ while(trans > 0L &&
+ (readSize = fileChannel.read(byteBuffer, this.position+position)) > 0) {
+ //adjust counters and buffer limit
+ if(readSize < trans) {
+ trans -= readSize;
+ position += readSize;
+ byteBuffer.flip();
+ } else {
+ //We can read more than we need if the actualCount is not multiple
+ //of the byteBuffer size and file is big enough. In that case we cannot
+ //use flip method but we need to set buffer limit manually to trans.
+ byteBuffer.limit((int)trans);
+ byteBuffer.position(0);
+ position += trans;
+ trans = 0;
+ }
+
+ //write data to the target
+ while(byteBuffer.hasRemaining()) {
+ target.write(byteBuffer);
+ }
+
+ byteBuffer.clear();
+ }
+
+ return actualCount - trans;
+ }
+
+
+ @Override
+ public void releaseExternalResources() {
+ if (readaheadRequest != null) {
+ readaheadRequest.cancel();
+ }
+ super.releaseExternalResources();
+ }
+
+ /**
+ * Call when the transfer completes successfully so we can advise the OS that
+ * we don't need the region to be cached anymore.
+ */
+ public void transferSuccessful() {
+ if (manageOsCache && getCount() > 0) {
+ try {
+ NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
+ fd, getPosition(), getCount(),
+ NativeIO.POSIX.POSIX_FADV_DONTNEED);
+ } catch (Throwable t) {
+ LOG.warn("Failed to manage OS cache for " + identifier, t);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java
new file mode 100644
index 0000000..9a51ca0
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.shufflehandler;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.tez.runtime.library.common.Constants;
+import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
+import org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord;
+
+class IndexCache {
+
+ private final Configuration conf;
+ private final int totalMemoryAllowed;
+ private AtomicInteger totalMemoryUsed = new AtomicInteger();
+ private static final Log LOG = LogFactory.getLog(IndexCache.class);
+
+ private final ConcurrentHashMap<String,IndexInformation> cache =
+ new ConcurrentHashMap<String,IndexInformation>();
+
+ private final LinkedBlockingQueue<String> queue =
+ new LinkedBlockingQueue<String>();
+
+ public IndexCache(Configuration conf) {
+ this.conf = conf;
+ totalMemoryAllowed = 10 * 1024 * 1024;
+ LOG.info("IndexCache created with max memory = " + totalMemoryAllowed);
+ }
+
+ /**
+ * This method gets the index information for the given mapId and reduce.
+ * It reads the index file into cache if it is not already present.
+ * @param mapId
+ * @param reduce
+ * @param fileName The file to read the index information from if it is not
+ * already present in the cache
+ * @param expectedIndexOwner The expected owner of the index file
+ * @return The Index Information
+ * @throws IOException
+ */
+ public TezIndexRecord getIndexInformation(String mapId, int reduce,
+ Path fileName, String expectedIndexOwner)
+ throws IOException {
+
+ IndexInformation info = cache.get(mapId);
+
+ if (info == null) {
+ info = readIndexFileToCache(fileName, mapId, expectedIndexOwner);
+ } else {
+ synchronized(info) {
+ while (isUnderConstruction(info)) {
+ try {
+ info.wait();
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for construction", e);
+ }
+ }
+ }
+ LOG.debug("IndexCache HIT: MapId " + mapId + " found");
+ }
+
+ if (info.mapSpillRecord.size() == 0 ||
+ info.mapSpillRecord.size() <= reduce) {
+ throw new IOException("Invalid request " +
+ " Map Id = " + mapId + " Reducer = " + reduce +
+ " Index Info Length = " + info.mapSpillRecord.size());
+ }
+ return info.mapSpillRecord.getIndex(reduce);
+ }
+
+ private boolean isUnderConstruction(IndexInformation info) {
+ synchronized(info) {
+ return (null == info.mapSpillRecord);
+ }
+ }
+
+ private IndexInformation readIndexFileToCache(Path indexFileName,
+ String mapId,
+ String expectedIndexOwner)
+ throws IOException {
+ IndexInformation info;
+ IndexInformation newInd = new IndexInformation();
+ if ((info = cache.putIfAbsent(mapId, newInd)) != null) {
+ synchronized(info) {
+ while (isUnderConstruction(info)) {
+ try {
+ info.wait();
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for construction", e);
+ }
+ }
+ }
+ LOG.debug("IndexCache HIT: MapId " + mapId + " found");
+ return info;
+ }
+ LOG.debug("IndexCache MISS: MapId " + mapId + " not found") ;
+ TezSpillRecord tmp = null;
+ try {
+ tmp = new TezSpillRecord(indexFileName, conf, expectedIndexOwner);
+ } catch (Throwable e) {
+ tmp = new TezSpillRecord(0);
+ cache.remove(mapId);
+ throw new IOException("Error Reading IndexFile", e);
+ } finally {
+ synchronized (newInd) {
+ newInd.mapSpillRecord = tmp;
+ newInd.notifyAll();
+ }
+ }
+ queue.add(mapId);
+
+ if (totalMemoryUsed.addAndGet(newInd.getSize()) > totalMemoryAllowed) {
+ freeIndexInformation();
+ }
+ return newInd;
+ }
+
+ /**
+ * This method removes the map from the cache if index information for this
+ * map is loaded(size>0), index information entry in cache will not be
+ * removed if it is in the loading phrase(size=0), this prevents corruption
+ * of totalMemoryUsed. It should be called when a map output on this tracker
+ * is discarded.
+ * @param mapId The taskID of this map.
+ */
+ public void removeMap(String mapId) {
+ IndexInformation info = cache.get(mapId);
+ if (info == null || ((info != null) && isUnderConstruction(info))) {
+ return;
+ }
+ info = cache.remove(mapId);
+ if (info != null) {
+ totalMemoryUsed.addAndGet(-info.getSize());
+ if (!queue.remove(mapId)) {
+ LOG.warn("Map ID" + mapId + " not found in queue!!");
+ }
+ } else {
+ LOG.info("Map ID " + mapId + " not found in cache");
+ }
+ }
+
+ /**
+ * This method checks if cache and totolMemoryUsed is consistent.
+ * It is only used for unit test.
+ * @return True if cache and totolMemoryUsed is consistent
+ */
+ boolean checkTotalMemoryUsed() {
+ int totalSize = 0;
+ for (IndexInformation info : cache.values()) {
+ totalSize += info.getSize();
+ }
+ return totalSize == totalMemoryUsed.get();
+ }
+
+ /**
+ * Bring memory usage below totalMemoryAllowed.
+ */
+ private synchronized void freeIndexInformation() {
+ while (totalMemoryUsed.get() > totalMemoryAllowed) {
+ String s = queue.remove();
+ IndexInformation info = cache.remove(s);
+ if (info != null) {
+ totalMemoryUsed.addAndGet(-info.getSize());
+ }
+ }
+ }
+
+ private static class IndexInformation {
+ TezSpillRecord mapSpillRecord;
+
+ int getSize() {
+ return mapSpillRecord == null
+ ? 0
+ : mapSpillRecord.size() * Constants.MAP_OUTPUT_INDEX_RECORD_LENGTH;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/78d4ebc1/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java
new file mode 100644
index 0000000..cc82d74
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java
@@ -0,0 +1,840 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.shufflehandler;
+
+import static org.jboss.netty.buffer.ChannelBuffers.wrappedBuffer;
+import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
+import static org.jboss.netty.handler.codec.http.HttpMethod.GET;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.METHOD_NOT_ALLOWED;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.UNAUTHORIZED;
+import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+
+import javax.crypto.SecretKey;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.ReadaheadPool;
+import org.apache.hadoop.io.SecureIOUtils;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.MutableCounterInt;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.runtime.library.common.security.SecureShuffleUtils;
+import org.apache.tez.runtime.library.common.shuffle.orderedgrouped.ShuffleHeader;
+import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
+import org.jboss.netty.bootstrap.ServerBootstrap;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelPipelineFactory;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
+import org.jboss.netty.channel.group.ChannelGroup;
+import org.jboss.netty.channel.group.DefaultChannelGroup;
+import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
+import org.jboss.netty.handler.codec.frame.TooLongFrameException;
+import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
+import org.jboss.netty.handler.codec.http.HttpChunkAggregator;
+import org.jboss.netty.handler.codec.http.HttpHeaders;
+import org.jboss.netty.handler.codec.http.HttpRequest;
+import org.jboss.netty.handler.codec.http.HttpRequestDecoder;
+import org.jboss.netty.handler.codec.http.HttpResponse;
+import org.jboss.netty.handler.codec.http.HttpResponseEncoder;
+import org.jboss.netty.handler.codec.http.HttpResponseStatus;
+import org.jboss.netty.handler.codec.http.QueryStringDecoder;
+import org.jboss.netty.handler.ssl.SslHandler;
+import org.jboss.netty.handler.stream.ChunkedWriteHandler;
+import org.jboss.netty.util.CharsetUtil;
+
+public class ShuffleHandler {
+
+ private static final Log LOG = LogFactory.getLog(ShuffleHandler.class);
+
+ public static final String SHUFFLE_HANDLER_LOCAL_DIRS = "tez.shuffle.handler.local-dirs";
+
+ public static final String SHUFFLE_MANAGE_OS_CACHE = "mapreduce.shuffle.manage.os.cache";
+ public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true;
+
+ public static final String SHUFFLE_READAHEAD_BYTES = "mapreduce.shuffle.readahead.bytes";
+ public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES = 4 * 1024 * 1024;
+
+ // pattern to identify errors related to the client closing the socket early
+ // idea borrowed from Netty SslHandler
+ private static final Pattern IGNORABLE_ERROR_MESSAGE = Pattern.compile(
+ "^.*(?:connection.*reset|connection.*closed|broken.*pipe).*$",
+ Pattern.CASE_INSENSITIVE);
+
+ private int port;
+ private final ChannelFactory selector;
+ private final ChannelGroup accepted = new DefaultChannelGroup();
+ protected HttpPipelineFactory pipelineFact;
+ private final int sslFileBufferSize;
+ private final Configuration conf;
+
+ private final ConcurrentMap<String, Boolean> registeredApps = new ConcurrentHashMap<String, Boolean>();
+
+ /**
+ * Should the shuffle use posix_fadvise calls to manage the OS cache during
+ * sendfile
+ */
+ private final boolean manageOsCache;
+ private final int readaheadLength;
+ private final int maxShuffleConnections;
+ private final int shuffleBufferSize;
+ private final boolean shuffleTransferToAllowed;
+ private final ReadaheadPool readaheadPool = ReadaheadPool.getInstance();
+
+ private Map<String,String> userRsrc;
+ private JobTokenSecretManager secretManager;
+
+ // TODO Fix this for tez.
+ public static final String MAPREDUCE_SHUFFLE_SERVICEID =
+ "mapreduce_shuffle";
+
+ public static final String SHUFFLE_PORT_CONFIG_KEY = "tez.shuffle.port";
+ public static final int DEFAULT_SHUFFLE_PORT = 15551;
+
+ // TODO Change configs to remove mapreduce references.
+ public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED =
+ "mapreduce.shuffle.connection-keep-alive.enable";
+ public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false;
+
+ public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT =
+ "mapreduce.shuffle.connection-keep-alive.timeout";
+ public static final int DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT = 5; //seconds
+
+ public static final String SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
+ "mapreduce.shuffle.mapoutput-info.meta.cache.size";
+ public static final int DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
+ 1000;
+
+ public static final String CONNECTION_CLOSE = "close";
+
+ public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY =
+ "mapreduce.shuffle.ssl.file.buffer.size";
+
+ public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024;
+
+ public static final String MAX_SHUFFLE_CONNECTIONS = "mapreduce.shuffle.max.connections";
+ public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS = 0; // 0 implies no limit
+
+ public static final String MAX_SHUFFLE_THREADS = "mapreduce.shuffle.max.threads";
+ // 0 implies Netty default of 2 * number of available processors
+ public static final int DEFAULT_MAX_SHUFFLE_THREADS = 0;
+
+ public static final String SHUFFLE_BUFFER_SIZE =
+ "mapreduce.shuffle.transfer.buffer.size";
+ public static final int DEFAULT_SHUFFLE_BUFFER_SIZE = 128 * 1024;
+
+ public static final String SHUFFLE_TRANSFERTO_ALLOWED =
+ "mapreduce.shuffle.transferTo.allowed";
+ public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true;
+ public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED =
+ false;
+
+ final boolean connectionKeepAliveEnabled;
+ final int connectionKeepAliveTimeOut;
+ final int mapOutputMetaInfoCacheSize;
+ private static final AtomicBoolean started = new AtomicBoolean(false);
+ private static final AtomicBoolean initing = new AtomicBoolean(false);
+ private static ShuffleHandler INSTANCE;
+
+ @Metrics(about="Shuffle output metrics", context="mapred")
+ static class ShuffleMetrics implements ChannelFutureListener {
+ @Metric("Shuffle output in bytes")
+ MutableCounterLong shuffleOutputBytes;
+ @Metric("# of failed shuffle outputs")
+ MutableCounterInt shuffleOutputsFailed;
+ @Metric("# of succeeeded shuffle outputs")
+ MutableCounterInt shuffleOutputsOK;
+ @Metric("# of current shuffle connections")
+ MutableGaugeInt shuffleConnections;
+
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ if (future.isSuccess()) {
+ shuffleOutputsOK.incr();
+ } else {
+ shuffleOutputsFailed.incr();
+ }
+ shuffleConnections.decr();
+ }
+ }
+
+ public ShuffleHandler(Configuration conf) {
+ this.conf = conf;
+ manageOsCache = conf.getBoolean(SHUFFLE_MANAGE_OS_CACHE,
+ DEFAULT_SHUFFLE_MANAGE_OS_CACHE);
+
+ readaheadLength = conf.getInt(SHUFFLE_READAHEAD_BYTES,
+ DEFAULT_SHUFFLE_READAHEAD_BYTES);
+
+ maxShuffleConnections = conf.getInt(MAX_SHUFFLE_CONNECTIONS,
+ DEFAULT_MAX_SHUFFLE_CONNECTIONS);
+ int maxShuffleThreads = conf.getInt(MAX_SHUFFLE_THREADS,
+ DEFAULT_MAX_SHUFFLE_THREADS);
+ if (maxShuffleThreads == 0) {
+ maxShuffleThreads = 2 * Runtime.getRuntime().availableProcessors();
+ }
+
+ shuffleBufferSize = conf.getInt(SHUFFLE_BUFFER_SIZE,
+ DEFAULT_SHUFFLE_BUFFER_SIZE);
+
+ shuffleTransferToAllowed = conf.getBoolean(SHUFFLE_TRANSFERTO_ALLOWED,
+ (Shell.WINDOWS)?WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED:
+ DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED);
+
+ ThreadFactory bossFactory = new ThreadFactoryBuilder()
+ .setNameFormat("ShuffleHandler Netty Boss #%d")
+ .build();
+ ThreadFactory workerFactory = new ThreadFactoryBuilder()
+ .setNameFormat("ShuffleHandler Netty Worker #%d")
+ .build();
+
+ selector = new NioServerSocketChannelFactory(
+ Executors.newCachedThreadPool(bossFactory),
+ Executors.newCachedThreadPool(workerFactory),
+ maxShuffleThreads);
+
+ sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY,
+ DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE);
+ connectionKeepAliveEnabled =
+ conf.getBoolean(SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED,
+ DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED);
+ connectionKeepAliveTimeOut =
+ Math.max(1, conf.getInt(SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
+ DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT));
+ mapOutputMetaInfoCacheSize =
+ Math.max(1, conf.getInt(SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE,
+ DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE));
+
+ userRsrc = new ConcurrentHashMap<String,String>();
+ secretManager = new JobTokenSecretManager();
+ }
+
+
+ public void start() throws Exception {
+ ServerBootstrap bootstrap = new ServerBootstrap(selector);
+ try {
+ pipelineFact = new HttpPipelineFactory(conf);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ bootstrap.setPipelineFactory(pipelineFact);
+ port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
+ Channel ch = bootstrap.bind(new InetSocketAddress(port));
+ accepted.add(ch);
+ port = ((InetSocketAddress)ch.getLocalAddress()).getPort();
+ conf.set(SHUFFLE_PORT_CONFIG_KEY, Integer.toString(port));
+ pipelineFact.SHUFFLE.setPort(port);
+ LOG.info("TezShuffleHandler" + " listening on port " + port);
+ }
+
+ public static void initializeAndStart(Configuration conf) throws Exception {
+ if (!initing.getAndSet(true)) {
+ INSTANCE = new ShuffleHandler(conf);
+ INSTANCE.start();
+ started.set(true);
+ }
+ }
+
+ public static ShuffleHandler get() {
+ Preconditions.checkState(started.get(), "ShuffleHandler must be started before invoking started");
+ return INSTANCE;
+ }
+
+ /**
+ * Serialize the shuffle port into a ByteBuffer for use later on.
+ * @param port the port to be sent to the ApplciationMaster
+ * @return the serialized form of the port.
+ */
+ public static ByteBuffer serializeMetaData(int port) throws IOException {
+ //TODO these bytes should be versioned
+ DataOutputBuffer port_dob = new DataOutputBuffer();
+ port_dob.writeInt(port);
+ return ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength());
+ }
+
+ /**
+ * A helper function to deserialize the metadata returned by ShuffleHandler.
+ * @param meta the metadata returned by the ShuffleHandler
+ * @return the port the Shuffle Handler is listening on to serve shuffle data.
+ */
+ public static int deserializeMetaData(ByteBuffer meta) throws IOException {
+ //TODO this should be returning a class not just an int
+ DataInputByteBuffer in = new DataInputByteBuffer();
+ in.reset(meta);
+ int port = in.readInt();
+ return port;
+ }
+
+ /**
+ * A helper function to serialize the JobTokenIdentifier to be sent to the
+ * ShuffleHandler as ServiceData.
+ * @param jobToken the job token to be used for authentication of
+ * shuffle data requests.
+ * @return the serialized version of the jobToken.
+ */
+ public static ByteBuffer serializeServiceData(Token<JobTokenIdentifier> jobToken) throws IOException {
+ //TODO these bytes should be versioned
+ DataOutputBuffer jobToken_dob = new DataOutputBuffer();
+ jobToken.write(jobToken_dob);
+ return ByteBuffer.wrap(jobToken_dob.getData(), 0, jobToken_dob.getLength());
+ }
+
+ static Token<JobTokenIdentifier> deserializeServiceData(ByteBuffer secret) throws IOException {
+ DataInputByteBuffer in = new DataInputByteBuffer();
+ in.reset(secret);
+ Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>();
+ jt.readFields(in);
+ return jt;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public void registerApplication(String applicationIdString, Token<JobTokenIdentifier> appToken,
+ String user) {
+ Boolean registered = registeredApps.putIfAbsent(applicationIdString, Boolean.valueOf(true));
+ if (registered == null) {
+ recordJobShuffleInfo(applicationIdString, user, appToken);
+ }
+ }
+
+ public void unregisterApplication(String applicationIdString) {
+ removeJobShuffleInfo(applicationIdString);
+ }
+
+
+ public void stop() throws Exception {
+ accepted.close().awaitUninterruptibly(10, TimeUnit.SECONDS);
+ if (selector != null) {
+ ServerBootstrap bootstrap = new ServerBootstrap(selector);
+ bootstrap.releaseExternalResources();
+ }
+ if (pipelineFact != null) {
+ pipelineFact.destroy();
+ }
+ }
+
+ protected Shuffle getShuffle(Configuration conf) {
+ return new Shuffle(conf);
+ }
+
+
+ private void addJobToken(String appIdString, String user,
+ Token<JobTokenIdentifier> jobToken) {
+ String jobIdString = appIdString.replace("application", "job");
+ userRsrc.put(jobIdString, user);
+ secretManager.addTokenForJob(jobIdString, jobToken);
+ LOG.info("Added token for " + jobIdString);
+ }
+
+ private void recordJobShuffleInfo(String appIdString, String user,
+ Token<JobTokenIdentifier> jobToken) {
+ addJobToken(appIdString, user, jobToken);
+ }
+
+ private void removeJobShuffleInfo(String appIdString) {
+ secretManager.removeTokenForJob(appIdString);
+ userRsrc.remove(appIdString);
+ }
+
+ class HttpPipelineFactory implements ChannelPipelineFactory {
+
+ final Shuffle SHUFFLE;
+ private SSLFactory sslFactory;
+
+ public HttpPipelineFactory(Configuration conf) throws Exception {
+ SHUFFLE = getShuffle(conf);
+ // TODO Setup SSL Shuffle
+// if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY,
+// MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) {
+// LOG.info("Encrypted shuffle is enabled.");
+// sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf);
+// sslFactory.init();
+// }
+ }
+
+ public void destroy() {
+ if (sslFactory != null) {
+ sslFactory.destroy();
+ }
+ }
+
+ @Override
+ public ChannelPipeline getPipeline() throws Exception {
+ ChannelPipeline pipeline = Channels.pipeline();
+ if (sslFactory != null) {
+ pipeline.addLast("ssl", new SslHandler(sslFactory.createSSLEngine()));
+ }
+ pipeline.addLast("decoder", new HttpRequestDecoder());
+ pipeline.addLast("aggregator", new HttpChunkAggregator(1 << 16));
+ pipeline.addLast("encoder", new HttpResponseEncoder());
+ pipeline.addLast("chunking", new ChunkedWriteHandler());
+ pipeline.addLast("shuffle", SHUFFLE);
+ return pipeline;
+ // TODO factor security manager into pipeline
+ // TODO factor out encode/decode to permit binary shuffle
+ // TODO factor out decode of index to permit alt. models
+ }
+
+ }
+
+ class Shuffle extends SimpleChannelUpstreamHandler {
+
+ private final Configuration conf;
+ private final IndexCache indexCache;
+ private final LocalDirAllocator lDirAlloc =
+ new LocalDirAllocator(SHUFFLE_HANDLER_LOCAL_DIRS);
+ private int port;
+
+ public Shuffle(Configuration conf) {
+ this.conf = conf;
+ indexCache = new IndexCache(conf);
+ this.port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
+ }
+
+ public void setPort(int port) {
+ this.port = port;
+ }
+
+ private List<String> splitMaps(List<String> mapq) {
+ if (null == mapq) {
+ return null;
+ }
+ final List<String> ret = new ArrayList<String>();
+ for (String s : mapq) {
+ Collections.addAll(ret, s.split(","));
+ }
+ return ret;
+ }
+
+ @Override
+ public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent evt)
+ throws Exception {
+ if ((maxShuffleConnections > 0) && (accepted.size() >= maxShuffleConnections)) {
+ LOG.info(String.format("Current number of shuffle connections (%d) is " +
+ "greater than or equal to the max allowed shuffle connections (%d)",
+ accepted.size(), maxShuffleConnections));
+ evt.getChannel().close();
+ return;
+ }
+ accepted.add(evt.getChannel());
+ super.channelOpen(ctx, evt);
+
+ }
+
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt)
+ throws Exception {
+ HttpRequest request = (HttpRequest) evt.getMessage();
+ if (request.getMethod() != GET) {
+ sendError(ctx, METHOD_NOT_ALLOWED);
+ return;
+ }
+ // Check whether the shuffle version is compatible
+ if (!ShuffleHeader.DEFAULT_HTTP_HEADER_NAME.equals(
+ request.getHeader(ShuffleHeader.HTTP_HEADER_NAME))
+ || !ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION.equals(
+ request.getHeader(ShuffleHeader.HTTP_HEADER_VERSION))) {
+ sendError(ctx, "Incompatible shuffle request version", BAD_REQUEST);
+ }
+ final Map<String,List<String>> q =
+ new QueryStringDecoder(request.getUri()).getParameters();
+ final List<String> keepAliveList = q.get("keepAlive");
+ boolean keepAliveParam = false;
+ if (keepAliveList != null && keepAliveList.size() == 1) {
+ keepAliveParam = Boolean.valueOf(keepAliveList.get(0));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("KeepAliveParam : " + keepAliveList
+ + " : " + keepAliveParam);
+ }
+ }
+ final List<String> mapIds = splitMaps(q.get("map"));
+ final List<String> reduceQ = q.get("reduce");
+ final List<String> jobQ = q.get("job");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("RECV: " + request.getUri() +
+ "\n mapId: " + mapIds +
+ "\n reduceId: " + reduceQ +
+ "\n jobId: " + jobQ +
+ "\n keepAlive: " + keepAliveParam);
+ }
+
+ if (mapIds == null || reduceQ == null || jobQ == null) {
+ sendError(ctx, "Required param job, map and reduce", BAD_REQUEST);
+ return;
+ }
+ if (reduceQ.size() != 1 || jobQ.size() != 1) {
+ sendError(ctx, "Too many job/reduce parameters", BAD_REQUEST);
+ return;
+ }
+ int reduceId;
+ String jobId;
+ try {
+ reduceId = Integer.parseInt(reduceQ.get(0));
+ jobId = jobQ.get(0);
+ } catch (NumberFormatException e) {
+ sendError(ctx, "Bad reduce parameter", BAD_REQUEST);
+ return;
+ } catch (IllegalArgumentException e) {
+ sendError(ctx, "Bad job parameter", BAD_REQUEST);
+ return;
+ }
+ final String reqUri = request.getUri();
+ if (null == reqUri) {
+ // TODO? add upstream?
+ sendError(ctx, FORBIDDEN);
+ return;
+ }
+ HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
+ try {
+ verifyRequest(jobId, ctx, request, response,
+ new URL("http", "", this.port, reqUri));
+ } catch (IOException e) {
+ LOG.warn("Shuffle failure ", e);
+ sendError(ctx, e.getMessage(), UNAUTHORIZED);
+ return;
+ }
+
+ Map<String, MapOutputInfo> mapOutputInfoMap =
+ new HashMap<String, MapOutputInfo>();
+ Channel ch = evt.getChannel();
+ String user = userRsrc.get(jobId);
+
+ // $x/$user/appcache/$appId/output/$mapId
+ // TODO: Once Shuffle is out of NM, this can use MR APIs to convert
+ // between App and Job
+ String outputBasePathStr = getBaseLocation(jobId, user);
+
+ try {
+ populateHeaders(mapIds, outputBasePathStr, user, reduceId, request,
+ response, keepAliveParam, mapOutputInfoMap);
+ } catch(IOException e) {
+ ch.write(response);
+ LOG.error("Shuffle error in populating headers :", e);
+ String errorMessage = getErrorMessage(e);
+ sendError(ctx,errorMessage , INTERNAL_SERVER_ERROR);
+ return;
+ }
+ ch.write(response);
+ // TODO refactor the following into the pipeline
+ ChannelFuture lastMap = null;
+ for (String mapId : mapIds) {
+ try {
+ MapOutputInfo info = mapOutputInfoMap.get(mapId);
+ if (info == null) {
+ info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+ }
+ lastMap =
+ sendMapOutput(ctx, ch, user, mapId,
+ reduceId, info);
+ if (null == lastMap) {
+ sendError(ctx, NOT_FOUND);
+ return;
+ }
+ } catch (IOException e) {
+ LOG.error("Shuffle error :", e);
+ String errorMessage = getErrorMessage(e);
+ sendError(ctx,errorMessage , INTERNAL_SERVER_ERROR);
+ return;
+ }
+ }
+ lastMap.addListener(ChannelFutureListener.CLOSE);
+ }
+
+ private String getErrorMessage(Throwable t) {
+ StringBuffer sb = new StringBuffer(t.getMessage());
+ while (t.getCause() != null) {
+ sb.append(t.getCause().getMessage());
+ t = t.getCause();
+ }
+ return sb.toString();
+ }
+
+ private final String USERCACHE_CONSTANT = "usercache";
+ private final String APPCACHE_CONSTANT = "appcache";
+
+ private String getBaseLocation(String jobIdString, String user) {
+ String parts[] = jobIdString.split("_");
+ Preconditions.checkArgument(parts.length == 3, "Invalid jobId. Expecting 3 parts");
+ final ApplicationId appID =
+ ApplicationId.newInstance(Long.parseLong(parts[1]), Integer.parseInt(parts[2]));
+ final String baseStr =
+ USERCACHE_CONSTANT + "/" + user + "/"
+ + APPCACHE_CONSTANT + "/"
+ + ConverterUtils.toString(appID) + "/output" + "/";
+ return baseStr;
+ }
+
+ protected MapOutputInfo getMapOutputInfo(String base, String mapId,
+ int reduce, String user) throws IOException {
+ // Index file
+ Path indexFileName =
+ lDirAlloc.getLocalPathToRead(base + "/file.out.index", conf);
+ TezIndexRecord info =
+ indexCache.getIndexInformation(mapId, reduce, indexFileName, user);
+
+ Path mapOutputFileName =
+ lDirAlloc.getLocalPathToRead(base + "/file.out", conf);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(base + " : " + mapOutputFileName + " : " + indexFileName);
+ }
+ MapOutputInfo outputInfo = new MapOutputInfo(mapOutputFileName, info);
+ return outputInfo;
+ }
+
+ protected void populateHeaders(List<String> mapIds, String outputBaseStr,
+ String user, int reduce, HttpRequest request, HttpResponse response,
+ boolean keepAliveParam, Map<String, MapOutputInfo> mapOutputInfoMap)
+ throws IOException {
+
+ long contentLength = 0;
+ for (String mapId : mapIds) {
+ String base = outputBaseStr + mapId;
+ MapOutputInfo outputInfo = getMapOutputInfo(base, mapId, reduce, user);
+ if (mapOutputInfoMap.size() < mapOutputMetaInfoCacheSize) {
+ mapOutputInfoMap.put(mapId, outputInfo);
+ }
+ // Index file
+ Path indexFileName =
+ lDirAlloc.getLocalPathToRead(base + "/file.out.index", conf);
+ TezIndexRecord info =
+ indexCache.getIndexInformation(mapId, reduce, indexFileName, user);
+ ShuffleHeader header =
+ new ShuffleHeader(mapId, info.getPartLength(), info.getRawLength(), reduce);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ header.write(dob);
+
+ contentLength += info.getPartLength();
+ contentLength += dob.getLength();
+ }
+
+ // Now set the response headers.
+ setResponseHeaders(response, keepAliveParam, contentLength);
+ }
+
+ protected void setResponseHeaders(HttpResponse response,
+ boolean keepAliveParam, long contentLength) {
+ if (!connectionKeepAliveEnabled && !keepAliveParam) {
+ LOG.info("Setting connection close header...");
+ response.setHeader(HttpHeaders.Names.CONNECTION, CONNECTION_CLOSE);
+ } else {
+ response.setHeader(HttpHeaders.Names.CONTENT_LENGTH,
+ String.valueOf(contentLength));
+ response.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
+ response.setHeader(HttpHeaders.Values.KEEP_ALIVE, "timeout="
+ + connectionKeepAliveTimeOut);
+ LOG.info("Content Length in shuffle : " + contentLength);
+ }
+ }
+
+ class MapOutputInfo {
+ final Path mapOutputFileName;
+ final TezIndexRecord indexRecord;
+
+ MapOutputInfo(Path mapOutputFileName, TezIndexRecord indexRecord) {
+ this.mapOutputFileName = mapOutputFileName;
+ this.indexRecord = indexRecord;
+ }
+ }
+
+ protected void verifyRequest(String appid, ChannelHandlerContext ctx,
+ HttpRequest request, HttpResponse response, URL requestUri)
+ throws IOException {
+ SecretKey tokenSecret = secretManager.retrieveTokenSecret(appid);
+ if (null == tokenSecret) {
+ LOG.info("Request for unknown token " + appid);
+ throw new IOException("could not find jobid");
+ }
+ // string to encrypt
+ String enc_str = SecureShuffleUtils.buildMsgFrom(requestUri);
+ // hash from the fetcher
+ String urlHashStr =
+ request.getHeader(SecureShuffleUtils.HTTP_HEADER_URL_HASH);
+ if (urlHashStr == null) {
+ LOG.info("Missing header hash for " + appid);
+ throw new IOException("fetcher cannot be authenticated");
+ }
+ if (LOG.isDebugEnabled()) {
+ int len = urlHashStr.length();
+ LOG.debug("verifying request. enc_str=" + enc_str + "; hash=..." +
+ urlHashStr.substring(len-len/2, len-1));
+ }
+ // verify - throws exception
+ SecureShuffleUtils.verifyReply(urlHashStr, enc_str, tokenSecret);
+ // verification passed - encode the reply
+ String reply =
+ SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8),
+ tokenSecret);
+ response.setHeader(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH, reply);
+ // Put shuffle version into http header
+ response.setHeader(ShuffleHeader.HTTP_HEADER_NAME,
+ ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
+ response.setHeader(ShuffleHeader.HTTP_HEADER_VERSION,
+ ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
+ if (LOG.isDebugEnabled()) {
+ int len = reply.length();
+ LOG.debug("Fetcher request verfied. enc_str=" + enc_str + ";reply=" +
+ reply.substring(len-len/2, len-1));
+ }
+ }
+
+ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch,
+ String user, String mapId, int reduce, MapOutputInfo mapOutputInfo)
+ throws IOException {
+ final TezIndexRecord info = mapOutputInfo.indexRecord;
+ final ShuffleHeader header =
+ new ShuffleHeader(mapId, info.getPartLength(), info.getRawLength(), reduce);
+ final DataOutputBuffer dob = new DataOutputBuffer();
+ header.write(dob);
+ ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+ final File spillfile =
+ new File(mapOutputInfo.mapOutputFileName.toString());
+ RandomAccessFile spill;
+ try {
+ spill = SecureIOUtils.openForRandomRead(spillfile, "r", user, null);
+ } catch (FileNotFoundException e) {
+ LOG.info(spillfile + " not found");
+ return null;
+ }
+ ChannelFuture writeFuture;
+ if (ch.getPipeline().get(SslHandler.class) == null) {
+ final FadvisedFileRegion partition = new FadvisedFileRegion(spill,
+ info.getStartOffset(), info.getPartLength(), manageOsCache, readaheadLength,
+ readaheadPool, spillfile.getAbsolutePath(),
+ shuffleBufferSize, shuffleTransferToAllowed);
+ writeFuture = ch.write(partition);
+ writeFuture.addListener(new ChannelFutureListener() {
+ // TODO error handling; distinguish IO/connection failures,
+ // attribute to appropriate spill output
+ @Override
+ public void operationComplete(ChannelFuture future) {
+ if (future.isSuccess()) {
+ partition.transferSuccessful();
+ }
+ partition.releaseExternalResources();
+ }
+ });
+ } else {
+ // HTTPS cannot be done with zero copy.
+ final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill,
+ info.getStartOffset(), info.getPartLength(), sslFileBufferSize,
+ manageOsCache, readaheadLength, readaheadPool,
+ spillfile.getAbsolutePath());
+ writeFuture = ch.write(chunk);
+ }
+ return writeFuture;
+ }
+
+ protected void sendError(ChannelHandlerContext ctx,
+ HttpResponseStatus status) {
+ sendError(ctx, "", status);
+ }
+
+ protected void sendError(ChannelHandlerContext ctx, String message,
+ HttpResponseStatus status) {
+ HttpResponse response = new DefaultHttpResponse(HTTP_1_1, status);
+ response.setHeader(CONTENT_TYPE, "text/plain; charset=UTF-8");
+ // Put shuffle version into http header
+ response.setHeader(ShuffleHeader.HTTP_HEADER_NAME,
+ ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
+ response.setHeader(ShuffleHeader.HTTP_HEADER_VERSION,
+ ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
+ response.setContent(
+ ChannelBuffers.copiedBuffer(message, CharsetUtil.UTF_8));
+
+ // Close the connection as soon as the error message is sent.
+ ctx.getChannel().write(response).addListener(ChannelFutureListener.CLOSE);
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e)
+ throws Exception {
+ Channel ch = e.getChannel();
+ Throwable cause = e.getCause();
+ if (cause instanceof TooLongFrameException) {
+ sendError(ctx, BAD_REQUEST);
+ return;
+ } else if (cause instanceof IOException) {
+ if (cause instanceof ClosedChannelException) {
+ LOG.debug("Ignoring closed channel error", cause);
+ return;
+ }
+ String message = String.valueOf(cause.getMessage());
+ if (IGNORABLE_ERROR_MESSAGE.matcher(message).matches()) {
+ LOG.debug("Ignoring client socket close", cause);
+ return;
+ }
+ }
+
+ LOG.error("Shuffle error: ", cause);
+ if (ch.isConnected()) {
+ LOG.error("Shuffle error " + e);
+ sendError(ctx, INTERNAL_SERVER_ERROR);
+ }
+ }
+ }
+}