You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2015/02/18 23:40:48 UTC
[01/23] tez git commit: TEZ-2031. Tez UI: horizontal scrollbars do
not appear in tables,
causing them to look truncated. (Sreenath Somarajapuram via hitesh)
Repository: tez
Updated Branches:
refs/heads/TEZ-2003 4e8ee895e -> d4134b23d (forced update)
TEZ-2031. Tez UI: horizontal scrollbars do not appear in tables, causing them to look truncated. (Sreenath Somarajapuram via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/bc6a490d
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/bc6a490d
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/bc6a490d
Branch: refs/heads/TEZ-2003
Commit: bc6a490df579761dfb765fd3eaed718bf94922bc
Parents: 54bd104
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Feb 12 11:24:00 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Feb 12 11:24:00 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../app/scripts/components/extended-table.js | 25 ++++++++++++++++++++
tez-ui/src/main/webapp/app/styles/main.less | 12 +++++++++-
3 files changed, 37 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/bc6a490d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9f4b878..b33c454 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -61,6 +61,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2031. Tez UI: horizontal scrollbars do not appear in tables, causing them to look truncated.
TEZ-2073. SimpleHistoryLoggingService cannot be read by log aggregation (umask)
TEZ-2078. Tez UI: Task logs url use in-progress url causing various errors.
TEZ-2077. Tez UI: No diagnostics on Task Attempt Details page if task attempt failed.
http://git-wip-us.apache.org/repos/asf/tez/blob/bc6a490d/tez-ui/src/main/webapp/app/scripts/components/extended-table.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/extended-table.js b/tez-ui/src/main/webapp/app/scripts/components/extended-table.js
index 0999176..dd72935 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/extended-table.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/extended-table.js
@@ -18,6 +18,14 @@
App.ExTable = Ember.Namespace.create();
+Ember.Table.BodyTableContainer.reopen({
+ width: Ember.computed.alias('tableComponent._tableContainerWidth'),
+ didInsertElement: function () {
+ this._super();
+ this.$().unbind();
+ }
+});
+
App.ExTable.FilterTextField = Em.TextField.extend({
classNames: ['filter'],
classNameBindings: ['isPopulated','isInputDirty:input-dirty'],
@@ -161,12 +169,19 @@ App.ExTable.ColumnDefinition = Ember.Table.ColumnDefinition.extend({
App.ExTable.TableComponent = Ember.Table.EmberTableComponent.extend({
layoutName: 'components/extended-table/extable',
filters: {},
+ styleBindings: ['height', 'width'],
hasFilter: true,
minFilterHeight: 30, //TODO: less changes
enableContentSelection: false,
selectionMode: 'none',
+ width: function () {
+ return Math.max(this.get('_width'), this._getTotalWidth(this.get('tableColumns')));
+ }.property('tableColumns', '_tableColumnsWidth', '_width'),
+
+ _tableContainerWidth: Ember.computed.alias('width'),
+
actions: {
filterUpdated: function(columnDef, value) {
var filterID = columnDef.get('filterID');
@@ -177,6 +192,13 @@ App.ExTable.TableComponent = Ember.Table.EmberTableComponent.extend({
},
},
+ updateLayout: function () {
+ if ((this.get('_state') || this.get('state')) !== 'inDOM') {
+ return;
+ }
+ return this.doForceFillColumns();
+ },
+
doForceFillColumns: function() {
var additionWidthPerColumn, availableContentWidth, columnsToResize, contentWidth, fixedColumnsWidth, remainingWidth, tableColumns, totalWidth;
totalWidth = this.get('_width');
@@ -189,6 +211,9 @@ App.ExTable.TableComponent = Ember.Table.EmberTableComponent.extend({
remainingWidth = availableContentWidth - contentWidth;
columnsToResize = tableColumns.filterProperty('canAutoResize');
+ if(totalWidth < contentWidth) {
+ return [];
+ }
additionWidthPerColumn = Math.floor(remainingWidth / columnsToResize.length);
if(availableContentWidth <= this._getTotalWidth(tableColumns, 'minWidth')) {
return columnsToResize;
http://git-wip-us.apache.org/repos/asf/tez/blob/bc6a490d/tez-ui/src/main/webapp/app/styles/main.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/main.less b/tez-ui/src/main/webapp/app/styles/main.less
index 37251cc..2730af6 100644
--- a/tez-ui/src/main/webapp/app/styles/main.less
+++ b/tez-ui/src/main/webapp/app/styles/main.less
@@ -455,7 +455,17 @@ div.indent {
}
.table-container {
- min-height: 380px;
+ min-height: 100px;
+ width: 100%;
+ overflow: auto;
+ border-right: 1px solid @border-lite;
+ border-left: 1px solid @border-lite;
+ .ember-table-table-block {
+ overflow: visible !important;
+ }
+ .ember-table-scroll-container {
+ display: none;
+ }
}
.input-dirty {
[10/23] tez git commit: TEZ-2102. Tez UI: DAG view has hidden edges,
dragging DAG by holding vertex causes unintended click. (Sreenath
Somarajapuram via hitesh)
Posted by ss...@apache.org.
TEZ-2102. Tez UI: DAG view has hidden edges, dragging DAG by holding vertex causes unintended click. (Sreenath Somarajapuram via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a5a56659
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a5a56659
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a5a56659
Branch: refs/heads/TEZ-2003
Commit: a5a5665916acbd504bb095761b920df9ff36185e
Parents: 8e138a2
Author: Hitesh Shah <hi...@apache.org>
Authored: Sat Feb 14 08:53:21 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Sat Feb 14 08:53:21 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../components/dag-view/data-processor.js | 16 +++++------
.../scripts/components/dag-view/graph-view.js | 30 ++++++++++++++++----
tez-ui/src/main/webapp/app/styles/dag-view.less | 4 +--
4 files changed, 35 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/a5a56659/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 33f9614..fd20d20 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -64,6 +64,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2102. Tez UI: DAG view has hidden edges, dragging DAG by holding vertex causes unintended click.
TEZ-2101. Tez UI: Issues on displaying a table.
TEZ-2092. Tez UI history url handler injects spurious trailing slash.
TEZ-2098. Tez UI: Dag details should be the default page for dag, fix invalid time entries for failed Vertices.
http://git-wip-us.apache.org/repos/asf/tez/blob/a5a56659/tez-ui/src/main/webapp/app/scripts/components/dag-view/data-processor.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/dag-view/data-processor.js b/tez-ui/src/main/webapp/app/scripts/components/dag-view/data-processor.js
index fae5a38..362a602 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/dag-view/data-processor.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/dag-view/data-processor.js
@@ -74,7 +74,7 @@
* + Other custom properties including data to be displayed
* }
* ]
- * maxDepth, inputCount
+ * maxDepth, leafCount
* }
*
* Data Nodes:
@@ -579,27 +579,27 @@ App.DagViewComponent.dataProcessor = (function (){
*/
function _getGraphDetails(tree) {
var maxDepth = 0,
- inputCount = 0,
+ leafCount = 0,
links = _getLinks(tree);
tree.ifForEach('children', function (child) {
var details = _getGraphDetails(child);
maxDepth = Math.max(maxDepth, details.maxDepth);
-
- if(child.type == types.INPUT) {
- inputCount++;
- }
- inputCount += details.inputCount;
+ leafCount += details.leafCount;
links.push.apply(links, details.links);
});
+ if(!tree.get('children')) {
+ leafCount++;
+ }
+
return {
tree: tree,
links: links,
maxDepth: maxDepth + 1,
- inputCount: inputCount
+ leafCount: leafCount
};
}
http://git-wip-us.apache.org/repos/asf/tez/blob/a5a56659/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js b/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
index 02c04e6..88c11f7 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
@@ -86,6 +86,7 @@ App.DagViewComponent.graphView = (function (){
hSpacing: 180, // Horizontal spacing between nodes
vSpacing: 70, // Vertical spacing between nodes
depthSpacing: 180, // In leftToRight depthSpacing = hSpacing
+ linkDelta: 30, // Used for links starting and ending at the same point
projector: function (x, y) { // Converts coordinate based on current orientation
return {x: y, y: x};
},
@@ -96,6 +97,7 @@ App.DagViewComponent.graphView = (function (){
hSpacing: 120,
vSpacing: 100,
depthSpacing: 100, // In topToBottom depthSpacing = vSpacing
+ linkDelta: 15,
projector: function (x, y) {
return {x: x, y: y};
},
@@ -218,7 +220,7 @@ App.DagViewComponent.graphView = (function (){
* @param d {VertexDataNode}
*/
function _addStatusBar(node, d) {
- var group = node.append('g');
+ var group = node.append('g'),
statusIcon = App.Helpers.misc.getStatusClassForEntity(d.get('data'));
group.attr('class', 'status-bar');
@@ -486,6 +488,14 @@ App.DagViewComponent.graphView = (function (){
}
/**
+ * Callback for mousedown & mousemove interactions. To disable click on drag
+ * @param d {DataNode} Data of the clicked element
+ */
+ function _onMouse(d) {
+ d3.select(this).on('click', d3.event.type == 'mousedown' ? _onClick : null);
+ }
+
+ /**
* Double click event handler.
* @param d {DataNode} Data of the clicked element
*/
@@ -530,8 +540,8 @@ App.DagViewComponent.graphView = (function (){
sV = 45,
mY -= 50;
if(sX == tX) {
- sX += 30,
- tX -= 30;
+ sX += _layout.linkDelta,
+ tX -= _layout.linkDelta;
}
}
sH = Math.abs(sX - tX) * 1.1;
@@ -575,7 +585,8 @@ App.DagViewComponent.graphView = (function (){
.on({
mouseover: _onMouseOver,
mouseout: _tip.hide,
- click: _onClick,
+ mousedown: _onMouse,
+ mousemove: _onMouse,
dblclick: _onDblclick
})
.style('opacity', 1e-6)
@@ -786,7 +797,14 @@ App.DagViewComponent.graphView = (function (){
* @param layout {Object} One of the values defined in LAYOUTS object
*/
function _setLayout(layout) {
- var dimention = layout.projector(_data.inputCount, _data.maxDepth - 1);
+ var leafCount = _data.leafCount,
+ dimention;
+
+ // If count is even dummy will be replaced by output, so output would no more be leaf
+ if(_data.tree.get('children.length') % 2 == 0) {
+ leafCount--;
+ }
+ dimention = layout.projector(leafCount, _data.maxDepth - 1);
_layout = layout;
@@ -818,7 +836,7 @@ App.DagViewComponent.graphView = (function (){
_tip.init($(element).find('.tool-tip'), _svg);
_treeData = data.tree,
- _treeData.x0 = _height / 2,
+ _treeData.x0 = 0,
_treeData.y0 = 0;
_panZoom = _attachPanZoom(_svg, _g);
http://git-wip-us.apache.org/repos/asf/tez/blob/a5a56659/tez-ui/src/main/webapp/app/styles/dag-view.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/dag-view.less b/tez-ui/src/main/webapp/app/styles/dag-view.less
index 0ad61e2..4cb6707 100644
--- a/tez-ui/src/main/webapp/app/styles/dag-view.less
+++ b/tez-ui/src/main/webapp/app/styles/dag-view.less
@@ -256,7 +256,7 @@
pointer-events: none;
display: none;
- max-width: 420px;
+ max-width: 620px;
.sub {
font-size: 10px;
@@ -288,7 +288,7 @@
td {
overflow: hidden;
white-space: nowrap;
- max-width: 200;
+ max-width: 300px;
}
td:nth-child(1) {
padding-right: 10px;
[07/23] tez git commit: TEZ-2101. Tez UI: Issues on displaying a
table. (Sreenath Somarajapuram via hitesh)
Posted by ss...@apache.org.
TEZ-2101. Tez UI: Issues on displaying a table. (Sreenath Somarajapuram via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ada11cc4
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ada11cc4
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ada11cc4
Branch: refs/heads/TEZ-2003
Commit: ada11cc4d4bfe65706d412f66a6e86a012f22e64
Parents: 6f0825d
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Feb 13 10:53:02 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Feb 13 10:53:02 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
tez-ui/src/main/webapp/app/scripts/helpers/misc.js | 2 ++
tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js | 6 +++++-
tez-ui/src/main/webapp/app/styles/main.less | 2 +-
4 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/ada11cc4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ec631b3..9f33a85 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -62,6 +62,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2101. Tez UI: Issues on displaying a table.
TEZ-2092. Tez UI history url handler injects spurious trailing slash.
TEZ-2098. Tez UI: Dag details should be the default page for dag, fix invalid time entries for failed Vertices.
TEZ-2024. TaskFinishedEvent may not be logged in recovery.
http://git-wip-us.apache.org/repos/asf/tez/blob/ada11cc4/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
index 1bbb677..50663a8 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
@@ -17,6 +17,8 @@
App.Helpers.misc = {
getStatusClassForEntity: function(dag) {
+ if(!dag) return '';
+
var st = dag.get('status');
switch(st) {
case 'FAILED':
http://git-wip-us.apache.org/repos/asf/tez/blob/ada11cc4/tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js b/tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js
index f576edf..749be3a 100644
--- a/tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js
+++ b/tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js
@@ -29,7 +29,11 @@ App.DagViewView = Ember.View.extend({
},
didInsertElement: function() {
- $(window).resize(this.setHeight.bind(this));
+ $(window).on('resize', this.setHeight);
this.setHeight();
},
+
+ willDestroyElement: function () {
+ $(window).off('resize', this.setHeight);
+ }
});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/ada11cc4/tez-ui/src/main/webapp/app/styles/main.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/main.less b/tez-ui/src/main/webapp/app/styles/main.less
index 2730af6..161c8f3 100644
--- a/tez-ui/src/main/webapp/app/styles/main.less
+++ b/tez-ui/src/main/webapp/app/styles/main.less
@@ -455,7 +455,7 @@ div.indent {
}
.table-container {
- min-height: 100px;
+ min-height: 20px;
width: 100%;
overflow: auto;
border-right: 1px solid @border-lite;
[23/23] tez git commit: TEZ-2090. Add tests for jobs running in
external services. (sseth)
Posted by ss...@apache.org.
TEZ-2090. Add tests for jobs running in external services. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/d4134b23
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/d4134b23
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/d4134b23
Branch: refs/heads/TEZ-2003
Commit: d4134b23dfeda970f50a9e4777bef9a6f5920766
Parents: a7a760f
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Feb 13 17:24:05 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Feb 18 14:40:12 2015 -0800
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
pom.xml | 6 +
.../apache/tez/dag/api/TezConfiguration.java | 2 +
.../apache/tez/dag/api/TaskCommunicator.java | 1 +
.../tez/dag/api/TaskCommunicatorContext.java | 3 +
.../tez/dag/app/TezTaskCommunicatorImpl.java | 42 +-
.../dag/app/rm/TaskSchedulerEventHandler.java | 2 +-
tez-ext-service-tests/pom.xml | 161 ++++
.../tez/dag/app/TezTestServiceCommunicator.java | 152 ++++
.../TezTestServiceContainerLauncher.java | 144 ++++
.../TezTestServiceNoOpContainerLauncher.java | 66 ++
.../rm/TezTestServiceTaskSchedulerService.java | 347 ++++++++
.../TezTestServiceTaskCommunicatorImpl.java | 182 ++++
.../org/apache/tez/service/ContainerRunner.java | 27 +
.../tez/service/MiniTezTestServiceCluster.java | 163 ++++
.../service/TezTestServiceConfConstants.java | 41 +
.../TezTestServiceProtocolBlockingPB.java | 22 +
.../tez/service/impl/ContainerRunnerImpl.java | 512 +++++++++++
.../apache/tez/service/impl/TezTestService.java | 126 +++
.../impl/TezTestServiceProtocolClientImpl.java | 82 ++
.../impl/TezTestServiceProtocolServerImpl.java | 133 +++
.../tez/shufflehandler/FadvisedChunkedFile.java | 78 ++
.../tez/shufflehandler/FadvisedFileRegion.java | 160 ++++
.../apache/tez/shufflehandler/IndexCache.java | 199 +++++
.../tez/shufflehandler/ShuffleHandler.java | 840 +++++++++++++++++++
.../tez/tests/TestExternalTezServices.java | 183 ++++
.../org/apache/tez/util/ProtoConverters.java | 172 ++++
.../src/test/proto/TezDaemonProtocol.proto | 84 ++
.../src/test/resources/log4j.properties | 19 +
.../org/apache/tez/runtime/task/TezChild.java | 2 +-
.../apache/tez/runtime/task/TezTaskRunner.java | 2 +-
31 files changed, 3943 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index d7e4be5..975ce65 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -1,5 +1,6 @@
ALL CHANGES:
TEZ-2019. Temporarily allow the scheduler and launcher to be specified via configuration.
TEZ-2006. Task communication plane needs to be pluggable.
+ TEZ-2090. Add tests for jobs running in external services.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3396587..eb6bcad 100644
--- a/pom.xml
+++ b/pom.xml
@@ -158,6 +158,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-ext-service-tests</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
@@ -620,6 +625,7 @@
<module>tez-dag</module>
<module>tez-ui</module>
<module>tez-plugins</module>
+ <module>tez-ext-service-tests</module>
<module>tez-dist</module>
<module>docs</module>
</modules>
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index c35a853..6b0a6f8 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1140,6 +1140,8 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_AM_CONTAINER_LAUNCHER_CLASS = TEZ_AM_PREFIX + "container-launcher.class";
@ConfigurationScope(Scope.VERTEX)
public static final String TEZ_AM_TASK_SCHEDULER_CLASS = TEZ_AM_PREFIX + "task-scheduler.class";
+ @ConfigurationScope(Scope.VERTEX)
+ public static final String TEZ_AM_TASK_COMMUNICATOR_CLASS = TEZ_AM_PREFIX + "task-communicator.class";
// TODO only validate property here, value can also be validated if necessary
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
index 97f9c16..c9f85e0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
@@ -14,6 +14,7 @@
package org.apache.tez.dag.api;
+import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
index 9b2d889..41675fe 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
@@ -44,5 +44,8 @@ public interface TaskCommunicatorContext {
// TODO TEZ-2003 Move to vertex, taskIndex, version
void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId);
+ // TODO TEZ-2003 Add an API to register task failure - for example, a communication failure.
+ // This will have to take into consideration the TA_FAILED event
+
// TODO Eventually Add methods to report availability stats to the scheduler.
}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
index 5652937..258c927 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
@@ -74,16 +74,22 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
new ConcurrentHashMap<TaskAttempt, ContainerId>();
private final TezTaskUmbilicalProtocol taskUmbilical;
+ private final String tokenIdentifier;
+ private final Token<JobTokenIdentifier> sessionToken;
private InetSocketAddress address;
private Server server;
- private static final class ContainerInfo {
+ public static final class ContainerInfo {
- ContainerInfo(ContainerId containerId) {
+ ContainerInfo(ContainerId containerId, String host, int port) {
this.containerId = containerId;
+ this.host = host;
+ this.port = port;
}
- ContainerId containerId;
+ final ContainerId containerId;
+ public final String host;
+ public final int port;
TezHeartbeatResponse lastResponse = null;
TaskSpec taskSpec = null;
long lastRequestId = 0;
@@ -110,6 +116,8 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
super(TezTaskCommunicatorImpl.class.getName());
this.taskCommunicatorContext = taskCommunicatorContext;
this.taskUmbilical = new TezTaskUmbilicalProtocolImpl();
+ this.tokenIdentifier = this.taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString();
+ this.sessionToken = TokenCache.getSessionToken(taskCommunicatorContext.getCredentials());
}
@@ -130,9 +138,7 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
try {
JobTokenSecretManager jobTokenSecretManager =
new JobTokenSecretManager();
- Token<JobTokenIdentifier> sessionToken = TokenCache.getSessionToken(taskCommunicatorContext.getCredentials());
- jobTokenSecretManager.addTokenForJob(
- taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString(), sessionToken);
+ jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken);
server = new RPC.Builder(conf)
.setProtocol(TezTaskUmbilicalProtocol.class)
@@ -182,7 +188,7 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
@Override
public void registerRunningContainer(ContainerId containerId, String host, int port) {
- ContainerInfo oldInfo = registeredContainers.putIfAbsent(containerId, new ContainerInfo(containerId));
+ ContainerInfo oldInfo = registeredContainers.putIfAbsent(containerId, new ContainerInfo(containerId, host, port));
if (oldInfo != null) {
throw new TezUncheckedException("Multiple registrations for containerId: " + containerId);
}
@@ -230,9 +236,9 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
". Already registered to containerId: " + oldId);
}
}
-
}
+
@Override
public void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID) {
TaskAttempt taskAttempt = new TaskAttempt(taskAttemptID);
@@ -258,6 +264,18 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
return address;
}
+ protected String getTokenIdentifier() {
+ return tokenIdentifier;
+ }
+
+ protected Token<JobTokenIdentifier> getSessionToken() {
+ return sessionToken;
+ }
+
+ protected TaskCommunicatorContext getTaskCommunicatorContext() {
+ return taskCommunicatorContext;
+ }
+
public TezTaskUmbilicalProtocol getUmbilical() {
return this.taskUmbilical;
}
@@ -471,4 +489,12 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator {
return "TaskAttempt{" + "taskAttemptId=" + taskAttemptId + '}';
}
}
+
+ protected ContainerInfo getContainerInfo(ContainerId containerId) {
+ return registeredContainers.get(containerId);
+ }
+
+ protected ContainerId getContainerForAttempt(TezTaskAttemptID taskAttemptId) {
+ return attemptToContainerMap.get(new TaskAttempt(taskAttemptId));
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 97bd7c8..fb17300 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -350,7 +350,7 @@ public class TaskSchedulerEventHandler extends AbstractService
try {
Constructor<? extends TaskSchedulerService> ctor = taskSchedulerClazz
.getConstructor(TaskSchedulerAppCallback.class, AppContext.class, String.class,
- Integer.class, String.class, Configuration.class);
+ int.class, String.class, Configuration.class);
ctor.setAccessible(true);
TaskSchedulerService taskSchedulerService =
ctor.newInstance(this, appContext, host, port, trackingUrl, getConfig());
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml
new file mode 100644
index 0000000..37f68b1
--- /dev/null
+++ b/tez-ext-service-tests/pom.xml
@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>tez</artifactId>
+ <groupId>org.apache.tez</groupId>
+ <version>0.7.0-SNAPSHOT</version>
+ </parent>
+
+ <!-- TODO TEZ-2003 Merge this into the tez-tests module -->
+ <artifactId>tez-ext-service-tests</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-internals</artifactId>
+ </dependency>
+ <dependency>
+ <!-- Required for the ShuffleHandler -->
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-library</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-dag</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-tests</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <!--
+ Include all files in src/main/resources. By default, do not apply property
+ substitution (filtering=false), but do apply property substitution to
+ version-info.properties (filtering=true). This will substitute the
+ version information correctly, but prevent Maven from altering other files.
+ -->
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <excludes>
+ <exclude>tez-api-version-info.properties</exclude>
+ </excludes>
+ <filtering>false</filtering>
+ </resource>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>tez-api-version-info.properties</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
+ <imports>
+ <param>${basedir}/src/test/proto</param>
+ <param>${basedir}/../tez-api/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/test/proto</directory>
+ <includes>
+ <include>TezDaemonProtocol.proto</include>
+ </includes>
+ </source>
+ <output>${project.build.directory}/generated-test-sources/java</output>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java
new file mode 100644
index 0000000..ac50878
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app;
+
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.protobuf.Message;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.tez.service.TezTestServiceProtocolBlockingPB;
+import org.apache.tez.service.impl.TezTestServiceProtocolClientImpl;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto;
+
+public class TezTestServiceCommunicator extends AbstractService {
+
+ private final ConcurrentMap<String, TezTestServiceProtocolBlockingPB> hostProxies;
+ private final ListeningExecutorService executor;
+
+ // TODO Convert this into a singleton
+ public TezTestServiceCommunicator(int numThreads) {
+ super(TezTestServiceCommunicator.class.getSimpleName());
+ ExecutorService localExecutor = Executors.newFixedThreadPool(numThreads,
+ new ThreadFactoryBuilder().setNameFormat("TezTestServiceCommunicator #%2d").build());
+ this.hostProxies = new ConcurrentHashMap<String, TezTestServiceProtocolBlockingPB>();
+ executor = MoreExecutors.listeningDecorator(localExecutor);
+ }
+
+ @Override
+ public void serviceStop() {
+ executor.shutdownNow();
+ }
+
+
+ public void runContainer(RunContainerRequestProto request, String host, int port,
+ final ExecuteRequestCallback<RunContainerResponseProto> callback) {
+ ListenableFuture<RunContainerResponseProto> future = executor.submit(new RunContainerCallable(request, host, port));
+ Futures.addCallback(future, new FutureCallback<RunContainerResponseProto>() {
+ @Override
+ public void onSuccess(RunContainerResponseProto result) {
+ callback.setResponse(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ callback.indicateError(t);
+ }
+ });
+
+ }
+
+ public void submitWork(SubmitWorkRequestProto request, String host, int port,
+ final ExecuteRequestCallback<SubmitWorkResponseProto> callback) {
+ ListenableFuture<SubmitWorkResponseProto> future = executor.submit(new SubmitWorkCallable(request, host, port));
+ Futures.addCallback(future, new FutureCallback<SubmitWorkResponseProto>() {
+ @Override
+ public void onSuccess(SubmitWorkResponseProto result) {
+ callback.setResponse(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ callback.indicateError(t);
+ }
+ });
+
+ }
+
+
+ private class RunContainerCallable implements Callable<RunContainerResponseProto> {
+
+ final String hostname;
+ final int port;
+ final RunContainerRequestProto request;
+
+ private RunContainerCallable(RunContainerRequestProto request, String hostname, int port) {
+ this.hostname = hostname;
+ this.port = port;
+ this.request = request;
+ }
+
+ @Override
+ public RunContainerResponseProto call() throws Exception {
+ return getProxy(hostname, port).runContainer(null, request);
+ }
+ }
+
+ private class SubmitWorkCallable implements Callable<SubmitWorkResponseProto> {
+ final String hostname;
+ final int port;
+ final SubmitWorkRequestProto request;
+
+ private SubmitWorkCallable(SubmitWorkRequestProto request, String hostname, int port) {
+ this.hostname = hostname;
+ this.port = port;
+ this.request = request;
+ }
+
+ @Override
+ public SubmitWorkResponseProto call() throws Exception {
+ return getProxy(hostname, port).submitWork(null, request);
+ }
+ }
+
+ public interface ExecuteRequestCallback<T extends Message> {
+ void setResponse(T response);
+ void indicateError(Throwable t);
+ }
+
+ private TezTestServiceProtocolBlockingPB getProxy(String hostname, int port) {
+ String hostId = getHostIdentifier(hostname, port);
+
+ TezTestServiceProtocolBlockingPB proxy = hostProxies.get(hostId);
+ if (proxy == null) {
+ proxy = new TezTestServiceProtocolClientImpl(getConfig(), hostname, port);
+ TezTestServiceProtocolBlockingPB proxyOld = hostProxies.putIfAbsent(hostId, proxy);
+ if (proxyOld != null) {
+ // TODO Shutdown the new proxy.
+ proxy = proxyOld;
+ }
+ }
+ return proxy;
+ }
+
+ private String getHostIdentifier(String hostname, int port) {
+ return hostname + ":" + port;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
new file mode 100644
index 0000000..e83165b
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.launcher;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TezTestServiceCommunicator;
+import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
+import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
+import org.apache.tez.dag.app.rm.container.AMContainerEvent;
+import org.apache.tez.dag.app.rm.container.AMContainerEventLaunchFailed;
+import org.apache.tez.dag.app.rm.container.AMContainerEventLaunched;
+import org.apache.tez.dag.app.rm.container.AMContainerEventType;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
+import org.apache.tez.service.TezTestServiceConfConstants;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+
+public class TezTestServiceContainerLauncher extends AbstractService implements ContainerLauncher {
+
+ // TODO Support interruptability of tasks which haven't yet been launched.
+
+ // TODO May need multiple connections per target machine, depending upon how synchronization is handled in the RPC layer
+
+ static final Log LOG = LogFactory.getLog(TezTestServiceContainerLauncher.class);
+
+ private final AppContext context;
+ private final String tokenIdentifier;
+ private final TaskAttemptListener tal;
+ private final int servicePort;
+ private final TezTestServiceCommunicator communicator;
+ private final Clock clock;
+
+
+ // Configuration passed in here to set up final parameters
+ public TezTestServiceContainerLauncher(AppContext appContext, Configuration conf,
+ TaskAttemptListener tal) {
+ super(TezTestServiceContainerLauncher.class.getName());
+ this.clock = appContext.getClock();
+ int numThreads = conf.getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS,
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT);
+
+ this.servicePort = conf.getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1);
+ Preconditions.checkArgument(servicePort > 0,
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be set");
+ this.communicator = new TezTestServiceCommunicator(numThreads);
+ this.context = appContext;
+ this.tokenIdentifier = context.getApplicationID().toString();
+ this.tal = tal;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ communicator.init(conf);
+ }
+
+ @Override
+ public void serviceStart() {
+ communicator.start();
+ }
+
+ @Override
+ public void serviceStop() {
+ communicator.stop();
+ }
+
+ @Override
+ public void handle(NMCommunicatorEvent event) {
+ switch (event.getType()) {
+ case CONTAINER_LAUNCH_REQUEST:
+ final NMCommunicatorLaunchRequestEvent launchEvent = (NMCommunicatorLaunchRequestEvent) event;
+ RunContainerRequestProto runRequest = constructRunContainerRequest(launchEvent);
+ communicator.runContainer(runRequest, launchEvent.getNodeId().getHost(),
+ launchEvent.getNodeId().getPort(),
+ new TezTestServiceCommunicator.ExecuteRequestCallback<TezTestServiceProtocolProtos.RunContainerResponseProto>() {
+ @Override
+ public void setResponse(TezTestServiceProtocolProtos.RunContainerResponseProto response) {
+ LOG.info("Container: " + launchEvent.getContainerId() + " launch succeeded on host: " + launchEvent.getNodeId());
+ context.getEventHandler().handle(new AMContainerEventLaunched(launchEvent.getContainerId()));
+ ContainerLaunchedEvent lEvt = new ContainerLaunchedEvent(
+ launchEvent.getContainerId(), clock.getTime(), context.getApplicationAttemptId());
+ context.getHistoryHandler().handle(new DAGHistoryEvent(
+ null, lEvt));
+ }
+
+ @Override
+ public void indicateError(Throwable t) {
+ LOG.error("Failed to launch container: " + launchEvent.getContainer() + " on host: " + launchEvent.getNodeId(), t);
+ sendContainerLaunchFailedMsg(launchEvent.getContainerId(), t);
+ }
+ });
+ break;
+ case CONTAINER_STOP_REQUEST:
+ LOG.info("DEBUG: Ignoring STOP_REQUEST for event: " + event);
+ // that the container is actually done (normally received from RM)
+ // TODO Sending this out for an un-launched container is invalid
+ context.getEventHandler().handle(new AMContainerEvent(event.getContainerId(),
+ AMContainerEventType.C_NM_STOP_SENT));
+ break;
+ }
+ }
+
+ private RunContainerRequestProto constructRunContainerRequest(NMCommunicatorLaunchRequestEvent event) {
+ RunContainerRequestProto.Builder builder = RunContainerRequestProto.newBuilder();
+ builder.setAmHost(tal.getAddress().getHostName()).setAmPort(tal.getAddress().getPort());
+ builder.setAppAttemptNumber(event.getContainer().getId().getApplicationAttemptId().getAttemptId());
+ builder.setApplicationIdString(
+ event.getContainer().getId().getApplicationAttemptId().getApplicationId().toString());
+ builder.setTokenIdentifier(tokenIdentifier);
+ builder.setContainerIdString(event.getContainer().getId().toString());
+ builder.setCredentialsBinary(
+ ByteString.copyFrom(event.getContainerLaunchContext().getTokens()));
+ // TODO Avoid reading this from the environment
+ builder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
+ return builder.build();
+ }
+
+ @SuppressWarnings("unchecked")
+ void sendContainerLaunchFailedMsg(ContainerId containerId, Throwable t) {
+ context.getEventHandler().handle(new AMContainerEventLaunchFailed(containerId, t == null ? "" : t.getMessage()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java
new file mode 100644
index 0000000..8c8e486
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.launcher;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
+import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
+import org.apache.tez.dag.app.rm.container.AMContainerEvent;
+import org.apache.tez.dag.app.rm.container.AMContainerEventLaunched;
+import org.apache.tez.dag.app.rm.container.AMContainerEventType;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
+
+public class TezTestServiceNoOpContainerLauncher extends AbstractService implements ContainerLauncher {
+
+ static final Log LOG = LogFactory.getLog(TezTestServiceNoOpContainerLauncher.class);
+
+ private final AppContext context;
+ private final Clock clock;
+
+ public TezTestServiceNoOpContainerLauncher(AppContext appContext, Configuration conf,
+ TaskAttemptListener tal) {
+ super(TezTestServiceNoOpContainerLauncher.class.getName());
+ this.context = appContext;
+ this.clock = appContext.getClock();
+ }
+
+ @Override
+ public void handle(NMCommunicatorEvent event) {
+ switch(event.getType()) {
+ case CONTAINER_LAUNCH_REQUEST:
+ final NMCommunicatorLaunchRequestEvent launchEvent = (NMCommunicatorLaunchRequestEvent) event;
+ LOG.info("No-op launch for container: " + launchEvent.getContainerId() + " succeeded on host: " + launchEvent.getNodeId());
+ context.getEventHandler().handle(new AMContainerEventLaunched(launchEvent.getContainerId()));
+ ContainerLaunchedEvent lEvt = new ContainerLaunchedEvent(
+ launchEvent.getContainerId(), clock.getTime(), context.getApplicationAttemptId());
+ context.getHistoryHandler().handle(new DAGHistoryEvent(
+ null, lEvt));
+ break;
+ case CONTAINER_STOP_REQUEST:
+ LOG.info("DEBUG: Ignoring STOP_REQUEST for event: " + event);
+ context.getEventHandler().handle(new AMContainerEvent(event.getContainerId(),
+ AMContainerEventType.C_NM_STOP_SENT));
+ break;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
new file mode 100644
index 0000000..e3c18bf
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java
@@ -0,0 +1,347 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Ints;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.service.TezTestServiceConfConstants;
+
+
+// TODO Registration with RM - so that the AM is considered dead and restarted in the expiry interval - 10 minutes.
+
+public class TezTestServiceTaskSchedulerService extends TaskSchedulerService {
+
+ private static final Log LOG = LogFactory.getLog(TezTestServiceTaskSchedulerService.class);
+
+ private final ExecutorService appCallbackExecutor;
+ private final TaskSchedulerAppCallback appClientDelegate;
+ private final AppContext appContext;
+ private final List<String> serviceHosts;
+ private final ContainerFactory containerFactory;
+ private final Random random = new Random();
+ // Currently all services must be running on the same port.
+ private final int containerPort;
+
+ private final String clientHostname;
+ private final int clientPort;
+ private final String trackingUrl;
+ private final AtomicBoolean isStopped = new AtomicBoolean(false);
+ private final ConcurrentMap<Object, ContainerId> runningTasks =
+ new ConcurrentHashMap<Object, ContainerId>();
+
+ private final AMRMClientAsync<AMRMClient.ContainerRequest> amRmClient;
+
+ // Per instance
+ private final int memoryPerInstance;
+ private final int coresPerInstance;
+ private final int executorsPerInstance;
+
+ // Per Executor Thread
+ private final Resource resourcePerContainer;
+
+
+ public TezTestServiceTaskSchedulerService(TaskSchedulerAppCallback appClient,
+ AppContext appContext,
+ String clientHostname, int clientPort,
+ String trackingUrl,
+ Configuration conf) {
+ // Accepting configuration here to allow setting up fields as final
+ super(TezTestServiceTaskSchedulerService.class.getName());
+ this.appCallbackExecutor = createAppCallbackExecutorService();
+ this.appClientDelegate = createAppCallbackDelegate(appClient);
+ this.appContext = appContext;
+ this.serviceHosts = new LinkedList<String>();
+ this.containerFactory = new ContainerFactory(appContext);
+
+ this.memoryPerInstance = conf
+ .getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB, -1);
+ Preconditions.checkArgument(memoryPerInstance > 0,
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB +
+ " must be configured");
+
+ this.executorsPerInstance = conf.getInt(
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE,
+ -1);
+ Preconditions.checkArgument(executorsPerInstance > 0,
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE +
+ " must be configured");
+
+ this.coresPerInstance = conf
+ .getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_VCPUS_PER_INSTANCE,
+ executorsPerInstance);
+
+ this.containerPort = conf.getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1);
+ Preconditions.checkArgument(executorsPerInstance > 0,
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be configured");
+
+ this.clientHostname = clientHostname;
+ this.clientPort = clientPort;
+ this.trackingUrl = trackingUrl;
+
+ int memoryPerContainer = (int) (memoryPerInstance / (float) executorsPerInstance);
+ int coresPerContainer = (int) (coresPerInstance / (float) executorsPerInstance);
+ this.resourcePerContainer = Resource.newInstance(memoryPerContainer, coresPerContainer);
+ this.amRmClient = TezAMRMClientAsync.createAMRMClientAsync(5000, new FakeAmRmCallbackHandler());
+
+ String[] hosts = conf.getTrimmedStrings(TezTestServiceConfConstants.TEZ_TEST_SERVICE_HOSTS);
+ if (hosts == null || hosts.length == 0) {
+ hosts = new String[]{"localhost"};
+ }
+ for (String host : hosts) {
+ serviceHosts.add(host);
+ }
+
+ LOG.info("Running with configuration: " +
+ "memoryPerInstance=" + memoryPerInstance +
+ ", vcoresPerInstance=" + coresPerInstance +
+ ", executorsPerInstance=" + executorsPerInstance +
+ ", resourcePerContainerInferred=" + resourcePerContainer +
+ ", hosts=" + serviceHosts.toString());
+
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ amRmClient.init(conf);
+ }
+
+ @Override
+ public void serviceStart() {
+ amRmClient.start();
+ RegisterApplicationMasterResponse response;
+ try {
+ amRmClient.registerApplicationMaster(clientHostname, clientPort, trackingUrl);
+ } catch (YarnException e) {
+ throw new TezUncheckedException(e);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+
+ @Override
+ public void serviceStop() {
+ if (!this.isStopped.getAndSet(true)) {
+
+ try {
+ TaskSchedulerAppCallback.AppFinalStatus status = appClientDelegate.getFinalAppStatus();
+ amRmClient.unregisterApplicationMaster(status.exitStatus, status.exitMessage,
+ status.postCompletionTrackingUrl);
+ } catch (YarnException e) {
+ throw new TezUncheckedException(e);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ appCallbackExecutor.shutdownNow();
+ }
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ // TODO This needs information about all running executors, and the amount of memory etc available across the cluster.
+ return Resource
+ .newInstance(Ints.checkedCast(serviceHosts.size() * memoryPerInstance),
+ serviceHosts.size() * coresPerInstance);
+ }
+
+ @Override
+ public int getClusterNodeCount() {
+ return serviceHosts.size();
+ }
+
+ @Override
+ public void resetMatchLocalityForAllHeldContainers() {
+ }
+
+ @Override
+ public Resource getTotalResources() {
+ return Resource
+ .newInstance(Ints.checkedCast(serviceHosts.size() * memoryPerInstance),
+ serviceHosts.size() * coresPerInstance);
+ }
+
+ @Override
+ public void blacklistNode(NodeId nodeId) {
+ LOG.info("DEBUG: BlacklistNode not supported");
+ }
+
+ @Override
+ public void unblacklistNode(NodeId nodeId) {
+ LOG.info("DEBUG: unBlacklistNode not supported");
+ }
+
+ @Override
+ public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks,
+ Priority priority, Object containerSignature, Object clientCookie) {
+ String host = selectHost(hosts);
+ Container container =
+ containerFactory.createContainer(resourcePerContainer, priority, host, containerPort);
+ runningTasks.put(task, container.getId());
+ appClientDelegate.taskAllocated(task, clientCookie, container);
+ }
+
+
+ @Override
+ public void allocateTask(Object task, Resource capability, ContainerId containerId,
+ Priority priority, Object containerSignature, Object clientCookie) {
+ String host = selectHost(null);
+ Container container =
+ containerFactory.createContainer(resourcePerContainer, priority, host, containerPort);
+ runningTasks.put(task, container.getId());
+ appClientDelegate.taskAllocated(task, clientCookie, container);
+ }
+
+ @Override
+ public boolean deallocateTask(Object task, boolean taskSucceeded) {
+ ContainerId containerId = runningTasks.remove(task);
+ if (containerId == null) {
+ LOG.error("Could not determine ContainerId for task: " + task +
+ " . Could have hit a race condition. Ignoring." +
+ " The query may hang since this \"unknown\" container is now taking up a slot permanently");
+ return false;
+ }
+ appClientDelegate.containerBeingReleased(containerId);
+ return true;
+ }
+
+ @Override
+ public Object deallocateContainer(ContainerId containerId) {
+ LOG.info("DEBUG: Ignoring deallocateContainer for containerId: " + containerId);
+ return null;
+ }
+
+ @Override
+ public void setShouldUnregister() {
+
+ }
+
+ @Override
+ public boolean hasUnregistered() {
+ // Nothing to do. No registration involved.
+ return true;
+ }
+
+ private ExecutorService createAppCallbackExecutorService() {
+ return Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
+ .setNameFormat("TaskSchedulerAppCaller #%d").setDaemon(true).build());
+ }
+
+ private TaskSchedulerAppCallback createAppCallbackDelegate(
+ TaskSchedulerAppCallback realAppClient) {
+ return new TaskSchedulerAppCallbackWrapper(realAppClient,
+ appCallbackExecutor);
+ }
+
+ private String selectHost(String[] requestedHosts) {
+ String host = null;
+ if (requestedHosts != null && requestedHosts.length > 0) {
+ Arrays.sort(requestedHosts);
+ host = requestedHosts[0];
+ LOG.info("Selected host: " + host + " from requested hosts: " + Arrays.toString(requestedHosts));
+ } else {
+ host = serviceHosts.get(random.nextInt(serviceHosts.size()));
+ LOG.info("Selected random host: " + host + " since the request contained no host information");
+ }
+ return host;
+ }
+
+ static class ContainerFactory {
+ final AppContext appContext;
+ AtomicInteger nextId;
+
+ public ContainerFactory(AppContext appContext) {
+ this.appContext = appContext;
+ this.nextId = new AtomicInteger(2);
+ }
+
+ public Container createContainer(Resource capability, Priority priority, String hostname, int port) {
+ ApplicationAttemptId appAttemptId = appContext.getApplicationAttemptId();
+ ContainerId containerId = ContainerId.newInstance(appAttemptId, nextId.getAndIncrement());
+ NodeId nodeId = NodeId.newInstance(hostname, port);
+ String nodeHttpAddress = "hostname:0";
+
+ Container container = Container.newInstance(containerId,
+ nodeId,
+ nodeHttpAddress,
+ capability,
+ priority,
+ null);
+
+ return container;
+ }
+ }
+
+ private static class FakeAmRmCallbackHandler implements AMRMClientAsync.CallbackHandler {
+
+ @Override
+ public void onContainersCompleted(List<ContainerStatus> statuses) {
+
+ }
+
+ @Override
+ public void onContainersAllocated(List<Container> containers) {
+
+ }
+
+ @Override
+ public void onShutdownRequest() {
+
+ }
+
+ @Override
+ public void onNodesUpdated(List<NodeReport> updatedNodes) {
+
+ }
+
+ @Override
+ public float getProgress() {
+ return 0;
+ }
+
+ @Override
+ public void onError(Throwable e) {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
new file mode 100644
index 0000000..78cdcde
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.taskcomm;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.protobuf.ByteString;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
+import org.apache.tez.dag.app.TezTestServiceCommunicator;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto;
+import org.apache.tez.util.ProtoConverters;
+
+
+public class TezTestServiceTaskCommunicatorImpl extends TezTaskCommunicatorImpl {
+
+ private static final Log LOG = LogFactory.getLog(TezTestServiceTaskCommunicatorImpl.class);
+
+ private final TezTestServiceCommunicator communicator;
+ private final SubmitWorkRequestProto BASE_SUBMIT_WORK_REQUEST;
+ private final ConcurrentMap<String, ByteBuffer> credentialMap;
+
+ public TezTestServiceTaskCommunicatorImpl(
+ TaskCommunicatorContext taskCommunicatorContext) {
+ super(taskCommunicatorContext);
+ // TODO Maybe make this configurable
+ this.communicator = new TezTestServiceCommunicator(3);
+
+ SubmitWorkRequestProto.Builder baseBuilder = SubmitWorkRequestProto.newBuilder();
+
+ // TODO Avoid reading this from the environment
+ baseBuilder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
+ baseBuilder.setApplicationIdString(
+ taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString());
+ baseBuilder
+ .setAppAttemptNumber(taskCommunicatorContext.getApplicationAttemptId().getAttemptId());
+ baseBuilder.setTokenIdentifier(getTokenIdentifier());
+
+ BASE_SUBMIT_WORK_REQUEST = baseBuilder.build();
+
+ credentialMap = new ConcurrentHashMap<String, ByteBuffer>();
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ this.communicator.init(conf);
+ }
+
+ @Override
+ public void serviceStart() {
+ super.serviceStart();
+ this.communicator.start();
+ }
+
+ @Override
+ public void serviceStop() {
+ super.serviceStop();
+ }
+
+
+ @Override
+ public void registerRunningContainer(ContainerId containerId, String hostname, int port) {
+ super.registerRunningContainer(containerId, hostname, port);
+ }
+
+ @Override
+ public void registerContainerEnd(ContainerId containerId) {
+ super.registerContainerEnd(containerId);
+ }
+
+ @Override
+ public void registerRunningTaskAttempt(final ContainerId containerId, final TaskSpec taskSpec,
+ Map<String, LocalResource> additionalResources,
+ Credentials credentials,
+ boolean credentialsChanged) {
+ super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials,
+ credentialsChanged);
+ SubmitWorkRequestProto requestProto = null;
+ try {
+ requestProto = constructSubmitWorkRequest(containerId, taskSpec);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to construct request", e);
+ }
+ ContainerInfo containerInfo = getContainerInfo(containerId);
+ String host;
+ int port;
+ if (containerInfo != null) {
+ synchronized (containerInfo) {
+ host = containerInfo.host;
+ port = containerInfo.port;
+ }
+ } else {
+ // TODO Handle this properly
+ throw new RuntimeException("ContainerInfo not found for container: " + containerId +
+ ", while trying to launch task: " + taskSpec.getTaskAttemptID());
+ }
+ communicator.submitWork(requestProto, host, port,
+ new TezTestServiceCommunicator.ExecuteRequestCallback<SubmitWorkResponseProto>() {
+ @Override
+ public void setResponse(SubmitWorkResponseProto response) {
+ LOG.info("Successfully launched task: " + taskSpec.getTaskAttemptID());
+ getTaskCommunicatorContext()
+ .taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId);
+ }
+
+ @Override
+ public void indicateError(Throwable t) {
+ // TODO Handle this error. This is where an API on the context to indicate failure / rejection comes in.
+ LOG.info("Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
+ containerId, t);
+ }
+ });
+ }
+
+ @Override
+ public void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID) {
+ super.unregisterRunningTaskAttempt(taskAttemptID);
+ // Nothing else to do for now. The push API in the test does not support termination of a running task
+ }
+
+ private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId,
+ TaskSpec taskSpec) throws
+ IOException {
+ SubmitWorkRequestProto.Builder builder =
+ SubmitWorkRequestProto.newBuilder(BASE_SUBMIT_WORK_REQUEST);
+ builder.setContainerIdString(containerId.toString());
+ builder.setAmHost(getAddress().getHostName());
+ builder.setAmPort(getAddress().getPort());
+ Credentials taskCredentials = new Credentials();
+ // Credentials can change across DAGs. Ideally construct only once per DAG.
+ taskCredentials.addAll(getTaskCommunicatorContext().getCredentials());
+
+ ByteBuffer credentialsBinary = credentialMap.get(taskSpec.getDAGName());
+ if (credentialsBinary == null) {
+ credentialsBinary = serializeCredentials(getTaskCommunicatorContext().getCredentials());
+ credentialMap.putIfAbsent(taskSpec.getDAGName(), credentialsBinary.duplicate());
+ } else {
+ credentialsBinary = credentialsBinary.duplicate();
+ }
+ builder.setCredentialsBinary(ByteString.copyFrom(credentialsBinary));
+ builder.setTaskSpec(ProtoConverters.convertTaskSpecToProto(taskSpec));
+ return builder.build();
+ }
+
+ private ByteBuffer serializeCredentials(Credentials credentials) throws IOException {
+ Credentials containerCredentials = new Credentials();
+ containerCredentials.addAll(credentials);
+ DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
+ containerCredentials.writeTokenStorageToStream(containerTokens_dob);
+ ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0,
+ containerTokens_dob.getLength());
+ return containerCredentialsBuffer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java
new file mode 100644
index 0000000..2bca4ed
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service;
+
+import java.io.IOException;
+
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
+
+public interface ContainerRunner {
+
+ void queueContainer(RunContainerRequestProto request) throws IOException;
+ void submitWork(SubmitWorkRequestProto request) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
new file mode 100644
index 0000000..f47bd67
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.tez.service.impl.TezTestService;
+
+public class MiniTezTestServiceCluster extends AbstractService {
+
+ private static final Log LOG = LogFactory.getLog(MiniTezTestServiceCluster.class);
+
+ private final File testWorkDir;
+ private final long availableMemory;
+ private final int numExecutorsPerService;
+ private final String[] localDirs;
+ private final Configuration clusterSpecificConfiguration = new Configuration(false);
+
+ private TezTestService tezTestService;
+
+ public static MiniTezTestServiceCluster create(String clusterName, int numExecutorsPerService, long availableMemory, int numLocalDirs) {
+ return new MiniTezTestServiceCluster(clusterName, numExecutorsPerService, availableMemory, numLocalDirs);
+ }
+
+ // TODO Add support for multiple instances
+ private MiniTezTestServiceCluster(String clusterName, int numExecutorsPerService, long availableMemory, int numLocalDirs) {
+ super(clusterName + "_TezTestServerCluster");
+ Preconditions.checkArgument(numExecutorsPerService > 0);
+ Preconditions.checkArgument(availableMemory > 0);
+ Preconditions.checkArgument(numLocalDirs > 0);
+ String clusterNameTrimmed = clusterName.replace("$", "") + "_TezTestServerCluster";
+ File targetWorkDir = new File("target", clusterNameTrimmed);
+ try {
+ FileContext.getLocalFSFileContext().delete(
+ new Path(targetWorkDir.getAbsolutePath()), true);
+ } catch (Exception e) {
+ LOG.warn("Could not cleanup test workDir: " + targetWorkDir, e);
+ throw new RuntimeException("Could not cleanup test workDir: " + targetWorkDir, e);
+ }
+
+ if (Shell.WINDOWS) {
+ // The test working directory can exceed the maximum path length supported
+ // by some Windows APIs and cmd.exe (260 characters). To work around this,
+ // create a symlink in temporary storage with a much shorter path,
+ // targeting the full path to the test working directory. Then, use the
+ // symlink as the test working directory.
+ String targetPath = targetWorkDir.getAbsolutePath();
+ File link = new File(System.getProperty("java.io.tmpdir"),
+ String.valueOf(System.currentTimeMillis()));
+ String linkPath = link.getAbsolutePath();
+
+ try {
+ FileContext.getLocalFSFileContext().delete(new Path(linkPath), true);
+ } catch (IOException e) {
+ throw new YarnRuntimeException("could not cleanup symlink: " + linkPath, e);
+ }
+
+ // Guarantee target exists before creating symlink.
+ targetWorkDir.mkdirs();
+
+ Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
+ Shell.getSymlinkCommand(targetPath, linkPath));
+ try {
+ shexec.execute();
+ } catch (IOException e) {
+ throw new YarnRuntimeException(String.format(
+ "failed to create symlink from %s to %s, shell output: %s", linkPath,
+ targetPath, shexec.getOutput()), e);
+ }
+
+ this.testWorkDir = link;
+ } else {
+ this.testWorkDir = targetWorkDir;
+ }
+ this.numExecutorsPerService = numExecutorsPerService;
+ this.availableMemory = availableMemory;
+
+ // Setup Local Dirs
+ localDirs = new String[numLocalDirs];
+ for (int i = 0 ; i < numLocalDirs ; i++) {
+ File f = new File(testWorkDir, "localDir");
+ f.mkdirs();
+ LOG.info("Created localDir: " + f.getAbsolutePath());
+ localDirs[i] = f.getAbsolutePath();
+ }
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ tezTestService = new TezTestService(conf, numExecutorsPerService, availableMemory, localDirs);
+ tezTestService.init(conf);
+
+ }
+
+ @Override
+ public void serviceStart() {
+ tezTestService.start();
+
+ clusterSpecificConfiguration.set(TezTestServiceConfConstants.TEZ_TEST_SERVICE_HOSTS,
+ getServiceAddress().getHostName());
+ clusterSpecificConfiguration.setInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT,
+ getServiceAddress().getPort());
+
+ clusterSpecificConfiguration.setInt(
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE,
+ numExecutorsPerService);
+ clusterSpecificConfiguration.setLong(
+ TezTestServiceConfConstants.TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB, availableMemory);
+ }
+
+ @Override
+ public void serviceStop() {
+ tezTestService.stop();
+ }
+
+ /**
+ * return the address at which the service is listening
+ * @return host:port
+ */
+ public InetSocketAddress getServiceAddress() {
+ Preconditions.checkState(getServiceState() == STATE.STARTED);
+ return tezTestService.getListenerAddress();
+ }
+
+ public int getShufflePort() {
+ Preconditions.checkState(getServiceState() == STATE.STARTED);
+ return tezTestService.getShufflePort();
+ }
+
+ public Configuration getClusterSpecificConfiguration() {
+ Preconditions.checkState(getServiceState() == STATE.STARTED);
+ return clusterSpecificConfiguration;
+ }
+
+ // Mainly for verification
+ public int getNumSubmissions() {
+ return tezTestService.getNumSubmissions();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java
new file mode 100644
index 0000000..bf4a5bd
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service;
+
+public class TezTestServiceConfConstants {
+
+ private static final String TEZ_TEST_SERVICE_PREFIX = "tez.test.service.";
+
+ /** Number of executors per instance - used by the scheduler */
+ public static final String TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE = TEZ_TEST_SERVICE_PREFIX + "num.executors.per-instance";
+
+ /** Memory available per instance - used by the scheduler */
+ public static final String TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB = TEZ_TEST_SERVICE_PREFIX + "memory.per.instance.mb";
+
+ /** CPUs available per instance - used by the scheduler */
+ public static final String TEZ_TEST_SERVICE_VCPUS_PER_INSTANCE = TEZ_TEST_SERVICE_PREFIX + "vcpus.per.instance";
+
+
+ /** Hosts on which the service is running. Currently assuming a single port for all instances */
+ public static final String TEZ_TEST_SERVICE_HOSTS = TEZ_TEST_SERVICE_PREFIX + "hosts";
+
+ /** Port on which the Service(s) listen. Current a single port for all instances */
+ public static final String TEZ_TEST_SERVICE_RPC_PORT = TEZ_TEST_SERVICE_PREFIX + "rpc.port";
+
+ /** Number of threads to use in the AM to communicate with the external service */
+ public static final String TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS = TEZ_TEST_SERVICE_PREFIX + "communicator.num.threads";
+ public static final int TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT = 2;
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceProtocolBlockingPB.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceProtocolBlockingPB.java
new file mode 100644
index 0000000..1108f72
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceProtocolBlockingPB.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+
+@ProtocolInfo(protocolName = "org.apache.tez.service.TezTestServiceProtocolBlockingPB", protocolVersion = 1)
+public interface TezTestServiceProtocolBlockingPB extends TezTestServiceProtocolProtos.TezTestServiceProtocol.BlockingInterface {
+}
\ No newline at end of file
[05/23] tez git commit: TEZ-2075. Incompatible issue caused by
TEZ-1233 that TezConfiguration.TEZ_SITE_XML is made private (zjffdu)
Posted by ss...@apache.org.
TEZ-2075. Incompatible issue caused by TEZ-1233 that TezConfiguration.TEZ_SITE_XML is made private (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8e8405b3
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8e8405b3
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8e8405b3
Branch: refs/heads/TEZ-2003
Commit: 8e8405b370f647d9a3c23faf9ef7a6b88c99c572
Parents: b26fdfe
Author: Jeff Zhang <zj...@apache.org>
Authored: Fri Feb 13 22:45:08 2015 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Fri Feb 13 22:45:08 2015 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../src/main/java/org/apache/tez/dag/api/TezConfiguration.java | 2 +-
.../test/java/org/apache/tez/dag/api/TestTezConfiguration.java | 3 ++-
3 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/8e8405b3/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d62308b..df584a9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.7.0: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2075. Incompatible issue caused by TEZ-1233 that TezConfiguration.TEZ_SITE_XML is made private
TEZ-2082. Race condition in TaskAttemptListenerImpTezDag.getTask()
TEZ-1233. Allow configuration of framework parameters per vertex.
TEZ-2045. TaskAttemptListener should not pull Tasks from AMContainer. Instead these should be registered with the listener.
http://git-wip-us.apache.org/repos/asf/tez/blob/8e8405b3/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index cfd6426..0bf78f9 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -43,7 +43,7 @@ import com.google.common.annotations.VisibleForTesting;
@Public
public class TezConfiguration extends Configuration {
- private final static String TEZ_SITE_XML = "tez-site.xml";
+ public final static String TEZ_SITE_XML = "tez-site.xml";
private final static Log LOG = LogFactory.getLog(TezConfiguration.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/8e8405b3/tez-api/src/test/java/org/apache/tez/dag/api/TestTezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestTezConfiguration.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestTezConfiguration.java
index 4423f16..76d5f3c 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestTezConfiguration.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestTezConfiguration.java
@@ -57,7 +57,8 @@ public class TestTezConfiguration {
Class<?> c = TezConfiguration.class;
Set<String> expectedKeys = new HashSet<String>();
for (Field f : c.getFields()) {
- if (!f.getName().endsWith("DEFAULT") && f.getType() == String.class) {
+ if (!f.getName().endsWith("DEFAULT") && f.getType() == String.class
+ && !f.getName().equals("TEZ_SITE_XML")) {
String value = (String)f.get(null);
// not prefix
if (!value.endsWith(".")) {
[16/23] tez git commit: TEZ-2072. Add missing Private annotation to
createDAG in the DAG API class. (sseth)
Posted by ss...@apache.org.
TEZ-2072. Add missing Private annotation to createDAG in the DAG API
class. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0487e0ae
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0487e0ae
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0487e0ae
Branch: refs/heads/TEZ-2003
Commit: 0487e0ae5427afc4bb8a998055b53dc1bbdd1294
Parents: b756bc0
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Feb 18 10:40:02 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Feb 18 10:40:02 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
tez-api/src/main/java/org/apache/tez/dag/api/DAG.java | 5 ++++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/0487e0ae/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ff3adc8..2c643b0 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.7.0: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2072. Add missing Private annotation to createDAG in the DAG API class.
TEZ-2095. master branch fails to compile against hadoop-2.4.
TEZ-2093. Add events to MockDAGAppMaster and add e2e test for event routing
TEZ-2075. Incompatible issue caused by TEZ-1233 that TezConfiguration.TEZ_SITE_XML is made private
http://git-wip-us.apache.org/repos/asf/tez/blob/0487e0ae/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 914f946..1be6979 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -334,7 +334,9 @@ public class DAG {
public Map<String, LocalResource> getTaskLocalFiles() {
return commonTaskLocalFiles;
}
-
+
+ @Private
+ @VisibleForTesting
void checkAndInferOneToOneParallelism() {
// infer all 1-1 via dependencies
// collect all 1-1 edges where the source parallelism is set
@@ -664,6 +666,7 @@ public class DAG {
}
// create protobuf message describing DAG
+ @Private
public DAGPlan createDag(Configuration tezConf, Credentials extraCredentials,
Map<String, LocalResource> tezJarResources, LocalResource binaryConfig,
boolean tezLrsAsArchive) {
[12/23] tez git commit: TEZ-2112. Tez UI: fix offset calculation,
add home button to breadcrumbs. (Sreenath Somarajapuram via hitesh)
Posted by ss...@apache.org.
TEZ-2112. Tez UI: fix offset calculation, add home button to breadcrumbs. (Sreenath Somarajapuram via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1653d18b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1653d18b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1653d18b
Branch: refs/heads/TEZ-2003
Commit: 1653d18b8414842c318d542c13ff075b915aeea0
Parents: b6cb7f8
Author: Hitesh Shah <hi...@apache.org>
Authored: Tue Feb 17 11:07:56 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Tue Feb 17 11:07:56 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../webapp/app/scripts/views/dag-view-view.js | 20 ++++++++++++--------
tez-ui/src/main/webapp/app/styles/main.less | 7 +++++++
tez-ui/src/main/webapp/app/templates/dag.hbs | 2 +-
tez-ui/src/main/webapp/app/templates/dags.hbs | 2 +-
tez-ui/src/main/webapp/app/templates/error.hbs | 6 ++++++
tez-ui/src/main/webapp/app/templates/task.hbs | 2 +-
.../main/webapp/app/templates/task_attempt.hbs | 2 +-
tez-ui/src/main/webapp/app/templates/tasks.hbs | 2 +-
.../src/main/webapp/app/templates/tez-app.hbs | 2 +-
tez-ui/src/main/webapp/app/templates/vertex.hbs | 2 +-
11 files changed, 33 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6638e55..5458feb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -64,6 +64,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2112. Tez UI: fix offset calculation, add home button to breadcrumbs.
TEZ-2038. TEZ-UI DAG is always running in tez-ui when the app is failed but no DAGFinishedEvent is logged.
TEZ-2102. Tez UI: DAG view has hidden edges, dragging DAG by holding vertex causes unintended click.
TEZ-2101. Tez UI: Issues on displaying a table.
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js b/tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js
index 749be3a..f660572 100644
--- a/tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js
+++ b/tez-ui/src/main/webapp/app/scripts/views/dag-view-view.js
@@ -18,14 +18,18 @@
App.DagViewView = Ember.View.extend({
setHeight: function () {
- var container = $('.dag-view-component-container');
- container.height(
- Math.max(
- // 50 pixel is left at the bottom
- $(window).height() - container.offset().top - 50,
- 450 // Minimum dag view component container height
- )
- );
+ var container = $('.dag-view-component-container'),
+ offset;
+ if(container) {
+ offset = container.offset();
+ container.height(
+ Math.max(
+ // 50 pixel is left at the bottom
+ offset ? $(window).height() - offset.top - 50 : 0,
+ 450 // Minimum dag view component container height
+ )
+ );
+ }
},
didInsertElement: function() {
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/tez-ui/src/main/webapp/app/styles/main.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/main.less b/tez-ui/src/main/webapp/app/styles/main.less
index fef8c00..b29fa6e 100644
--- a/tez-ui/src/main/webapp/app/styles/main.less
+++ b/tez-ui/src/main/webapp/app/styles/main.less
@@ -104,10 +104,17 @@ body, html, body > .ember-view {
}
.breadcrumb {
+ padding: 5px 15px;
font-size: 16px;
.active .sub {
font-size: 12px;
}
+
+ .fa-home:before {
+ position: relative;
+ font-size: 1.5em;
+ top: 2px;
+ }
}
.navbar {
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/tez-ui/src/main/webapp/app/templates/dag.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/dag.hbs b/tez-ui/src/main/webapp/app/templates/dag.hbs
index b34aebe..c78de95 100644
--- a/tez-ui/src/main/webapp/app/templates/dag.hbs
+++ b/tez-ui/src/main/webapp/app/templates/dag.hbs
@@ -17,7 +17,7 @@
}}
<ul class="breadcrumb">
- <li>{{#link-to 'dags'}}All Dags{{/link-to}} <span class="divider"></span></li>
+ <li>{{#link-to 'application'}}<i class="fa fa-home"> All Dags</i>{{/link-to}} <span class="divider"></span></li>
<li class="active">Dag - <span class='sub'>{{name}}</span></li>
</ul>
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/tez-ui/src/main/webapp/app/templates/dags.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/dags.hbs b/tez-ui/src/main/webapp/app/templates/dags.hbs
index abe5ea5..7ea8924 100644
--- a/tez-ui/src/main/webapp/app/templates/dags.hbs
+++ b/tez-ui/src/main/webapp/app/templates/dags.hbs
@@ -17,7 +17,7 @@
}}
<ul class="breadcrumb">
- <li class="active">All Dags</li>
+ <li>{{#link-to 'application'}}<i class="fa fa-home"> All Dags</i>{{/link-to}} <span class="divider"></span></li>
</ul>
<div class='margin-small-vertical'>
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/tez-ui/src/main/webapp/app/templates/error.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/error.hbs b/tez-ui/src/main/webapp/app/templates/error.hbs
index 47e7cd5..3bd7bab 100644
--- a/tez-ui/src/main/webapp/app/templates/error.hbs
+++ b/tez-ui/src/main/webapp/app/templates/error.hbs
@@ -15,4 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
}}
+
+<ul class="breadcrumb">
+ <li>{{#link-to 'application'}}<i class="fa fa-home"> All Dags</i>{{/link-to}} <span class="divider"></span></li>
+ <li class="active">Error</span></li>
+</ul>
+
<h1>An Error Occurred</h1>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/tez-ui/src/main/webapp/app/templates/task.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/task.hbs b/tez-ui/src/main/webapp/app/templates/task.hbs
index cf528c0..0965df5 100644
--- a/tez-ui/src/main/webapp/app/templates/task.hbs
+++ b/tez-ui/src/main/webapp/app/templates/task.hbs
@@ -17,7 +17,7 @@
}}
<ul class="breadcrumb">
- <li>{{#link-to 'dags'}}All Dags{{/link-to}} <span class="divider"></span></li>
+ <li>{{#link-to 'application'}}<i class="fa fa-home"> All Dags</i>{{/link-to}} <span class="divider"></span></li>
<li>{{#link-to 'dag.vertices' dagID}}Dag{{/link-to}} <span class="divider"></span></li>
<li>{{#link-to 'vertex.tasks' vertexID}}Vertex{{/link-to}} <span class="divider"></span></li>
<li class="active">Task - <span class='sub'>{{id}}</span></li>
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/tez-ui/src/main/webapp/app/templates/task_attempt.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/task_attempt.hbs b/tez-ui/src/main/webapp/app/templates/task_attempt.hbs
index c8ffb3b..67a087d 100644
--- a/tez-ui/src/main/webapp/app/templates/task_attempt.hbs
+++ b/tez-ui/src/main/webapp/app/templates/task_attempt.hbs
@@ -17,7 +17,7 @@
}}
<ul class="breadcrumb">
- <li>{{#link-to 'dags'}}All Dags{{/link-to}} <span class="divider"></span></li>
+ <li>{{#link-to 'application'}}<i class="fa fa-home"> All Dags</i>{{/link-to}} <span class="divider"></span></li>
<li>{{#link-to 'dag.vertices' dagID}}Dag{{/link-to}} <span class="divider"></span></li>
<li>{{#link-to 'vertex.tasks' vertexID}}Vertex{{/link-to}} <span class="divider"></span></li>
<li>{{#link-to 'task.attempts' taskID}}Task{{/link-to}} <span class="divider"></span></li>
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/tez-ui/src/main/webapp/app/templates/tasks.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/tasks.hbs b/tez-ui/src/main/webapp/app/templates/tasks.hbs
index f8096c4..14324c2 100644
--- a/tez-ui/src/main/webapp/app/templates/tasks.hbs
+++ b/tez-ui/src/main/webapp/app/templates/tasks.hbs
@@ -17,7 +17,7 @@
}}
<ul class="breadcrumb">
- <li>{{#link-to 'dags'}}All Dags{{/link-to}} <span class="divider"></span></li>
+ <li>{{#link-to 'application'}}<i class="fa fa-home"> All Dags</i>{{/link-to}} <span class="divider"></span></li>
<li>{{#link-to 'dag.vertices' dagID}}Dag{{/link-to}} <span class="divider"></span></li>
<li class="active">Tasks</li>
</ul>
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/tez-ui/src/main/webapp/app/templates/tez-app.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/tez-app.hbs b/tez-ui/src/main/webapp/app/templates/tez-app.hbs
index 011fde7..d96103f 100644
--- a/tez-ui/src/main/webapp/app/templates/tez-app.hbs
+++ b/tez-ui/src/main/webapp/app/templates/tez-app.hbs
@@ -17,7 +17,7 @@
}}
<ul class="breadcrumb">
- <li>{{#link-to 'dags'}}All Dags{{/link-to}} <span class="divider"></span></li>
+ <li>{{#link-to 'application'}}<i class="fa fa-home"> All Dags</i>{{/link-to}} <span class="divider"></span></li>
<li class="active">Tez App - <span class='sub'>{{id}}</span></li>
</ul>
http://git-wip-us.apache.org/repos/asf/tez/blob/1653d18b/tez-ui/src/main/webapp/app/templates/vertex.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/vertex.hbs b/tez-ui/src/main/webapp/app/templates/vertex.hbs
index 923869c..86e0f70 100644
--- a/tez-ui/src/main/webapp/app/templates/vertex.hbs
+++ b/tez-ui/src/main/webapp/app/templates/vertex.hbs
@@ -17,7 +17,7 @@
}}
<ul class="breadcrumb">
- <li>{{#link-to 'dags'}}All Dags{{/link-to}} <span class="divider"></span></li>
+ <li>{{#link-to 'application'}}<i class="fa fa-home"> All Dags</i>{{/link-to}} <span class="divider"></span></li>
<li>{{#link-to 'dag.vertices' dagID}}Dag{{/link-to}} <span class="divider"></span></li>
<li class="active">Vertex - <span class='sub'>{{name}}</span></li>
</ul>
[14/23] tez git commit: TEZ-2106. TEZ UI: Display data load time,
and add a refresh button for items that can be refreshed. (Sreenath
Somarajapuram via hitesh)
Posted by ss...@apache.org.
TEZ-2106. TEZ UI: Display data load time, and add a refresh button for items that can be refreshed. (Sreenath Somarajapuram via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/dd44bdec
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/dd44bdec
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/dd44bdec
Branch: refs/heads/TEZ-2003
Commit: dd44bdec1394975e5e3dd0dd97532dd6ca3c10ce
Parents: ec7ad20
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Feb 18 10:03:56 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed Feb 18 10:03:56 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../app/scripts/components/counter-table.js | 2 +-
.../scripts/components/load-time-component.js | 34 +++++++++
.../controllers/dag-task-attempts-controller.js | 49 ++++++++++--
.../scripts/controllers/dag-view-controller.js | 18 +++++
.../app/scripts/controllers/dag_controller.js | 4 +
.../scripts/controllers/dag_index_controller.js | 73 +++++++++++-------
.../webapp/app/scripts/controllers/dag_tasks.js | 20 +++++
.../app/scripts/controllers/dag_vertices.js | 21 +++++-
.../app/scripts/controllers/dags_controller.js | 32 +++++---
.../scripts/controllers/shared-controllers.js | 24 ++++++
.../app/scripts/controllers/task_controller.js | 2 +-
.../controllers/task_index_controller.js | 2 +-
.../task_task_attempts_controller.js | 9 ++-
.../scripts/controllers/tez-app-controller.js | 2 +-
.../controllers/tez-app-dags-controller.js | 79 ++++++++++++--------
.../controllers/tez-app-index-controller.js | 47 ++++++++++++
.../scripts/controllers/vertex_controller.js | 4 +-
.../controllers/vertex_index_controller.js | 18 ++++-
.../vertex_task_attempts_controller.js | 50 +++++++++++--
.../controllers/vertex_tasks_controller.js | 19 +++++
.../src/main/webapp/app/scripts/helpers/misc.js | 13 ++++
.../app/scripts/mixins/model-refresh-mixin.js | 34 +++++++++
.../app/scripts/mixins/paginated_content.js | 20 +++--
.../app/scripts/models/abstract_entity.js | 21 ++++--
.../src/main/webapp/app/scripts/models/dag.js | 10 +--
tez-ui/src/main/webapp/app/styles/main.less | 35 ++++++++-
tez-ui/src/main/webapp/app/styles/shared.less | 4 +
.../webapp/app/templates/common/counters.hbs | 12 ++-
.../webapp/app/templates/common/swimlanes.hbs | 5 ++
.../app/templates/common/table-with-spinner.hbs | 7 +-
.../app/templates/components/counter-table.hbs | 6 +-
.../app/templates/components/load-time.hbs | 37 +++++++++
.../src/main/webapp/app/templates/dag/index.hbs | 10 ++-
.../src/main/webapp/app/templates/dag/view.hbs | 7 ++
.../webapp/app/templates/partials/configs.hbs | 7 +-
.../app/templates/partials/table-controls.hbs | 2 +-
.../main/webapp/app/templates/task/index.hbs | 10 ++-
.../webapp/app/templates/taskAttempt/index.hbs | 10 ++-
.../main/webapp/app/templates/tez-app/index.hbs | 11 ++-
.../webapp/app/templates/vertex/additionals.hbs | 7 ++
.../main/webapp/app/templates/vertex/index.hbs | 7 ++
42 files changed, 658 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cc481c2..a566a37 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -64,6 +64,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2106. TEZ UI: Display data load time, and add a refresh button for items that can be refreshed.
TEZ-2114. Tez UI: task/task attempt status is not available when its running.
TEZ-2112. Tez UI: fix offset calculation, add home button to breadcrumbs.
TEZ-2038. TEZ-UI DAG is always running in tez-ui when the app is failed but no DAGFinishedEvent is logged.
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/components/counter-table.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/counter-table.js b/tez-ui/src/main/webapp/app/scripts/components/counter-table.js
index ca58f52..781df32 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/counter-table.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/counter-table.js
@@ -49,7 +49,7 @@ App.CounterTableComponent = Em.Component.extend({
})
return filtered;
- }.property('data', 'nameFilter')
+ }.property('data', 'nameFilter', 'timeStamp')
});
Em.Handlebars.helper('counter-table-component', App.CounterTableComponent);
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/components/load-time-component.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/load-time-component.js b/tez-ui/src/main/webapp/app/scripts/components/load-time-component.js
new file mode 100644
index 0000000..648a395
--- /dev/null
+++ b/tez-ui/src/main/webapp/app/scripts/components/load-time-component.js
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+App.LoadTimeComponent = Em.Component.extend({
+ layoutName: 'components/load-time',
+
+ actions: {
+ refresh: function() {
+ this.sendAction('refresh');
+ }
+ },
+
+ displayTime: function() {
+ var time = this.get('time');
+ return time ? App.Helpers.date.dateFormat(time.getTime(), true) : null;
+ }.property('time')
+});
+
+Em.Handlebars.helper('load-time-component', App.LoadTimeComponent);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/dag-task-attempts-controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag-task-attempts-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag-task-attempts-controller.js
index 50df0bf..e2f5fad 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag-task-attempts-controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag-task-attempts-controller.js
@@ -40,6 +40,24 @@ App.DagTaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentM
this.setFiltersAndLoadEntities(filters);
},
+ load: function () {
+ var dag = this.get('controllers.dag.model'),
+ controller = this.get('controllers.dag'),
+ t = this;
+ t.set('loading', true);
+ dag.reload().then(function () {
+ return controller.loadAdditional(dag);
+ }).then(function () {
+ t.resetNavigation();
+ t.loadEntities();
+ }).catch(function(error){
+ Em.Logger.error(error);
+ var err = App.Helpers.misc.formatError(error, defaultErrMsg);
+ var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
+ App.Helpers.ErrorBar.getInstance().show(msg, err.details);
+ });
+ }.observes('count'),
+
actions : {
filterUpdated: function(filterID, value) {
// any validations required goes here.
@@ -52,9 +70,32 @@ App.DagTaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentM
}
},
+ updateLoading: function () {
+ var dagController = this.get('controllers.dag'),
+ model = this.get('controllers.dag.model'),
+ that = this,
+ dagStatus = that.get('controllers.dag.status');
+
+ dagController.loadAdditional(model).then(function () {
+ that.get('entities').forEach(function (attempt) {
+
+ var attemptStatus = App.Helpers.misc
+ .getFixedupDisplayStatus(attempt.get('status'));
+ if (attemptStatus == 'RUNNING' &&
+ App.Helpers.misc.isStatusInUnsuccessful(dagStatus)) {
+ attemptStatus = 'KILLED'
+ }
+ if (attemptStatus != attempt.get('status')) {
+ attempt.set('status', attemptStatus);
+ }
+ });
+
+ that.set('loading', false);
+ });
+ },
+
defaultColumnConfigs: function() {
var that = this;
- var dagStatus = this.get('controllers.dag.status');
return [
{
id: 'taskId',
@@ -126,11 +167,7 @@ App.DagTaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentM
{{view.cellContent.status}}</span>')
}),
getCellContent: function(row) {
- var status = App.Helpers.misc.getFixedupDisplayStatus(row.get('status'));
- if (status == 'RUNNING' &&
- App.Helpers.misc.isStatusInUnsuccessful(dagStatus)) {
- status = 'KILLED';
- }
+ var status = row.get('status');
return {
status: status,
statusIcon: App.Helpers.misc.getStatusClassForEntity(status)
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/dag-view-controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag-view-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag-view-controller.js
index 31753d1..a748cdf 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag-view-controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag-view-controller.js
@@ -37,6 +37,24 @@ App.DagViewController = Em.ObjectController.extend(App.PaginatedContentMixin, Ap
this.setFiltersAndLoadEntities(filters);
},
+ load: function () {
+ var dag = this.get('controllers.dag.model'),
+ controller = this.get('controllers.dag'),
+ t = this;
+ t.set('loading', true);
+ dag.reload().then(function () {
+ return controller.loadAdditional(dag);
+ }).then(function () {
+ t.resetNavigation();
+ t.loadEntities();
+ }).catch(function(error){
+ Em.Logger.error(error);
+ var err = App.Helpers.misc.formatError(error, defaultErrMsg);
+ var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
+ App.Helpers.ErrorBar.getInstance().show(msg, err.details);
+ });
+ },
+
actions: {
entityClicked: function (details) {
switch(details.type) {
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
index c8d9c87..b3f15be 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
@@ -25,8 +25,10 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
var that = this;
var loaders = [];
var applicationId = dag.get('applicationId');
+
if (dag.get('status') === 'RUNNING') {
// update the progress info if available. this need not block the UI
+ App.Helpers.misc.removeRecord(this.store, 'dagProgress', dag.get('id'));
var aminfoLoader = that.store.find('dagProgress', dag.get('id'), {
appId: applicationId,
dagIdx: dag.get('idx')
@@ -37,6 +39,7 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
});
loaders.push(aminfoLoader);
}
+ App.Helpers.misc.removeRecord(this.store, 'appDetail', applicationId);
var appDetailLoader = this.store.find('appDetail', applicationId)
.then(function(app){
dag.set('appDetail', app);
@@ -46,6 +49,7 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
}
dag.set('status', App.Helpers.misc.getRealStatus(dag.get('status'), app.get('appState'), app.get('finalAppStatus')));
}).catch(function(){});
+ App.Helpers.misc.removeRecord(this.store, 'tezApp', 'tez_' + applicationId);
var tezAppLoader = this.store.find('tezApp', 'tez_' + applicationId)
.then(function(app){
dag.set('tezApp', app);
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
index 3261787..7576149 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
@@ -17,12 +17,29 @@
*/
//TODO: watch individual counters.
-App.DagIndexController = Em.ObjectController.extend({
- controllerName: 'DagIndexController',
+App.DagIndexController = Em.ObjectController.extend(App.ModelRefreshMixin, {
+ controllerName: 'DagIndexController',
- taskIconStatus: function() {
- return App.Helpers.misc.getStatusClassForEntity(this.get('model.status'));
- }.property('id', 'status', 'counterGroups'),
+ needs: 'dag',
+
+ load: function () {
+ var dag = this.get('controllers.dag.model'),
+ controller = this.get('controllers.dag'),
+ t = this;
+ t.set('loading', true);
+ dag.reload().then(function () {
+ return controller.loadAdditional(dag);
+ }).catch(function(error){
+ Em.Logger.error(error);
+ var err = App.Helpers.misc.formatError(error, defaultErrMsg);
+ var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
+ App.Helpers.ErrorBar.getInstance().show(msg, err.details);
+ });
+ },
+
+ taskIconStatus: function() {
+ return App.Helpers.misc.getStatusClassForEntity(this.get('model.status'));
+ }.property('id', 'status', 'counterGroups'),
progressStr: function() {
var pct;
@@ -32,29 +49,29 @@ App.DagIndexController = Em.ObjectController.extend({
return pct;
}.property('id', 'status', 'progress'),
- totalTasks: function() {
- return App.Helpers.misc.getCounterValueForDag(this.get('counterGroups'),
- this.get('id'), 'org.apache.tez.common.counters.DAGCounter', 'TOTAL_LAUNCHED_TASKS')
- }.property('id', 'counterGroups'),
-
- sucessfulTasks: function() {
- return App.Helpers.misc.getCounterValueForDag(this.get('counterGroups'), this.get('id'),
- 'org.apache.tez.common.counters.DAGCounter', 'NUM_SUCCEEDED_TASKS')
- }.property('id', 'counterGroups'),
-
- failedTasks: function() {
- return App.Helpers.misc.getCounterValueForDag(this.get('counterGroups'), this.get('id'),
- 'org.apache.tez.common.counters.DAGCounter', 'NUM_FAILED_TASKS')
- }.property('id', 'counterGroups'),
-
- killedTasks: function() {
- return App.Helpers.misc.getCounterValueForDag(this.get('counterGroups'), this.get('id'),
- 'org.apache.tez.common.counters.DAGCounter', 'NUM_KILLED_TASKS')
- }.property('id', 'counterGroups'),
-
- hasFailedTasks: function() {
- return this.get('failedTasks') > 0;
- }.property('id', 'counterGroups'),
+ totalTasks: function() {
+ return App.Helpers.misc.getCounterValueForDag(this.get('counterGroups'),
+ this.get('id'), 'org.apache.tez.common.counters.DAGCounter', 'TOTAL_LAUNCHED_TASKS')
+ }.property('id', 'counterGroups'),
+
+ sucessfulTasks: function() {
+ return App.Helpers.misc.getCounterValueForDag(this.get('counterGroups'), this.get('id'),
+ 'org.apache.tez.common.counters.DAGCounter', 'NUM_SUCCEEDED_TASKS')
+ }.property('id', 'counterGroups'),
+
+ failedTasks: function() {
+ return App.Helpers.misc.getCounterValueForDag(this.get('counterGroups'), this.get('id'),
+ 'org.apache.tez.common.counters.DAGCounter', 'NUM_FAILED_TASKS')
+ }.property('id', 'counterGroups'),
+
+ killedTasks: function() {
+ return App.Helpers.misc.getCounterValueForDag(this.get('counterGroups'), this.get('id'),
+ 'org.apache.tez.common.counters.DAGCounter', 'NUM_KILLED_TASKS')
+ }.property('id', 'counterGroups'),
+
+ hasFailedTasks: function() {
+ return this.get('failedTasks') > 0;
+ }.property('id', 'counterGroups'),
failedTasksLink: function() {
return '#tasks?status=FAILED&parentType=TEZ_DAG_ID&parentID=' + this.get('id');
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
index 93b9234..2b1304d 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
@@ -48,6 +48,24 @@ App.DagTasksController = Em.ObjectController.extend(App.PaginatedContentMixin, A
this.setFiltersAndLoadEntities(filters);
},
+ load: function () {
+ var dag = this.get('controllers.dag.model'),
+ controller = this.get('controllers.dag'),
+ t = this;
+ t.set('loading', true);
+ dag.reload().then(function () {
+ return controller.loadAdditional(dag);
+ }).then(function () {
+ t.resetNavigation();
+ t.loadEntities();
+ }).catch(function(error){
+ Em.Logger.error(error);
+ var err = App.Helpers.misc.formatError(error, defaultErrMsg);
+ var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
+ App.Helpers.ErrorBar.getInstance().show(msg, err.details);
+ });
+ }.observes('count'),
+
loadEntities: function() {
var that = this,
store = this.get('store'),
@@ -56,6 +74,7 @@ App.DagTasksController = Em.ObjectController.extend(App.PaginatedContentMixin, A
var defaultErrMsg = 'Error while loading tasks. could not connect to %@'
.fmt(App.env.timelineBaseUrl);
+ that.set('loading', true);
store.unloadAll(childEntityType);
store.findQuery(childEntityType, this.getFilterProperties())
.then(function(entities){
@@ -76,6 +95,7 @@ App.DagTasksController = Em.ObjectController.extend(App.PaginatedContentMixin, A
task.get('attempts.lastObject');
if (!!taskAttemptId) {
// Pivot attempt selection logic
+ App.Helpers.misc.removeRecord(store, 'taskAttempt', taskAttemptId);
fetcher = store.find('taskAttempt', taskAttemptId);
fetcher.then(function (attempt) {
task.set('pivotAttempt', attempt);
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
index f63524a..44ffd1a 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
@@ -44,10 +44,28 @@ App.DagVerticesController = Em.ObjectController.extend(App.PaginatedContentMixin
this.setFiltersAndLoadEntities(filters);
},
+ load: function () {
+ var dag = this.get('controllers.dag.model'),
+ controller = this.get('controllers.dag'),
+ t = this;
+ t.set('loading', true);
+ dag.reload().then(function () {
+ return controller.loadAdditional(dag);
+ }).then(function () {
+ t.resetNavigation();
+ t.loadEntities();
+ }).catch(function(error){
+ Em.Logger.error(error);
+ var err = App.Helpers.misc.formatError(error, defaultErrMsg);
+ var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
+ App.Helpers.ErrorBar.getInstance().show(msg, err.details);
+ });
+ }.observes('count'),
+
loadAdditional: function() {
var defer = Em.RSVP.defer();
- var that = this;
+ var that = this,
vertices = this.get('entities');
var dagStatus = this.get('controllers.dag.status');
@@ -64,6 +82,7 @@ App.DagVerticesController = Em.ObjectController.extend(App.PaginatedContentMixin
return item.get('id').split('_').splice(-1).pop();
});
if (runningVerticesIdx.length > 0) {
+ this.store.unloadAll('vertexProgress');
this.store.findQuery('vertexProgress', {
metadata: {
appId: that.get('applicationId'),
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
index 35a0ecc..880378a 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
@@ -65,33 +65,42 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
var that = this,
store = this.get('store'),
childEntityType = this.get('childEntityType'),
- fetcher;
+ fetcher,
+ record;
var defaultErrMsg = 'Error while loading dag info. could not connect to %@'.fmt(App.env.timelineBaseUrl);
+ that.set('loading', true);
store.unloadAll(childEntityType);
+ store.unloadAll('dagProgress');
+
store.findQuery(childEntityType, this.getFilterProperties()).then(function(entities){
var loaders = [];
- that.set('entities', entities);
entities.forEach(function (dag) {
var appId = dag.get('applicationId');
if(appId) {
// Pivot attempt selection logic
- fetcher = store.find('appDetail', appId);
- fetcher.then(function (app) {
+ record = store.getById('appDetail', appId);
+ if(record && !App.Helpers.misc.isStatusInUnsuccessful(record.get('appState'))) {
+ store.unloadRecord(record);
+ }
+ fetcher = store.find('appDetail', appId).then(function (app) {
dag.set('appDetail', app);
if (dag.get('status') === 'RUNNING') {
- dag.set('status', App.Helpers.misc.getRealStatus(dag.get('status'), app.get('appState'),
- app.get('finalAppStatus')));
+ dag.set('status', App.Helpers.misc.getRealStatus(
+ dag.get('status'),
+ app.get('appState'),
+ app.get('finalAppStatus')
+ ));
+ App.Helpers.misc.removeRecord(store, 'tezApp', 'tez_' + appId);
}
+ return store.find('tezApp', 'tez_' + appId).then(function (app) {
+ dag.set('tezApp', app);
+ });
});
loaders.push(fetcher);
//Load tezApp details
- fetcher = store.find('tezApp', 'tez_' + appId);
- fetcher.then(function (app) {
- dag.set('tezApp', app);
- });
- loaders.push(fetcher);
if (dag.get('status') === 'RUNNING') {
+ App.Helpers.misc.removeRecord(store, 'dagProgress', dag.get('id'));
amInfoFetcher = store.find('dagProgress', dag.get('id'), {
appId: dag.get('applicationId'),
dagIdx: dag.get('idx')
@@ -107,6 +116,7 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
}
});
Em.RSVP.allSettled(loaders).then(function(){
+ that.set('entities', entities);
that.set('loading', false);
});
}).catch(function(error){
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/shared-controllers.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/shared-controllers.js b/tez-ui/src/main/webapp/app/scripts/controllers/shared-controllers.js
new file mode 100644
index 0000000..70d5e8c
--- /dev/null
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/shared-controllers.js
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+App.DagCountersController =
+ App.VertexCountersController =
+ App.TaskCountersController =
+ App.TaskAttemptCountersController =
+
+ Em.ObjectController.extend(App.ModelRefreshMixin);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/task_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/task_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/task_controller.js
index ac4b22b..4f33b3b 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/task_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/task_controller.js
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-App.TaskController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
+App.TaskController = Em.ObjectController.extend(App.Helpers.DisplayHelper, App.ModelRefreshMixin, {
controllerName: 'TaskController',
pageTitle: 'Task',
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/task_index_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/task_index_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/task_index_controller.js
index 7f3eeb3..2c526c4 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/task_index_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/task_index_controller.js
@@ -17,7 +17,7 @@
*/
//TODO: watch individual counters.
-App.TaskIndexController = Em.ObjectController.extend({
+App.TaskIndexController = Em.ObjectController.extend(App.ModelRefreshMixin, {
controllerName: 'TaskIndexController',
taskStatus: function() {
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js
index 0749e00..b400375 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js
@@ -48,10 +48,8 @@ App.TaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentMixi
.fmt(childEntityType, App.env.timelineBaseUrl);
that.set('loading', true);
-
this.get('store').unloadAll(childEntityType);
this.get('store').findQuery(childEntityType, this.getFilterProperties()).then(function(entities){
- that.set('entities', entities);
var loaders = [];
try {
var loader = Em.tryInvoke(that, 'loadAdditional');
@@ -62,8 +60,10 @@ App.TaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentMixi
Em.Logger.error("Exception invoking additional load", error);
}
+ App.Helpers.misc.removeRecord(that.store, 'dag', that.get('controllers.task.dagID'));
var appDetailFetcher = that.store.find('dag', that.get('controllers.task.dagID')).
then(function (dag) {
+ App.Helpers.misc.removeRecord(that.store, 'appDetail', dag.get('applicationId'));
return that.store.find('appDetail', dag.get('applicationId'));
}).
then(function(appDetail) {
@@ -74,7 +74,8 @@ App.TaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentMixi
});
loaders.push(appDetailFetcher);
Em.RSVP.allSettled(loaders).then(function(){
- that.set('loading', false);
+ that.set('entities', entities);
+ that.set('loading', false);
});
}).catch(function(error){
Em.Logger.error(error);
@@ -253,7 +254,7 @@ App.TaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentMixi
});
-App.TaskAttemptIndexController = Em.ObjectController.extend({
+App.TaskAttemptIndexController = Em.ObjectController.extend(App.ModelRefreshMixin, {
controllerName: 'TaskAttemptIndexController',
taskAttemptStatus: function() {
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-controller.js
index 3df83fd..1941b3f 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-controller.js
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-App.TezAppController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
+App.TezAppController = Em.ObjectController.extend(App.Helpers.DisplayHelper, App.ModelRefreshMixin, {
controllerName: 'AppController',
pageTitle: 'App',
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
index 151438e..5d1ae0b 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
@@ -48,48 +48,61 @@ App.TezAppDagsController = Em.ObjectController.extend(App.PaginatedContentMixin,
},
loadEntities: function() {
- var that = this;
- var childEntityType = this.get('childEntityType');
- var defaultErrMsg = 'Error while loading %@. could not connect to %@'
- .fmt(childEntityType, App.env.timelineBaseUrl);
-
+ var that = this,
+ store = this.get('store'),
+ childEntityType = this.get('childEntityType'),
+ fetcher,
+ record;
+ var defaultErrMsg = 'Error while loading dag info. could not connect to %@'.fmt(App.env.timelineBaseUrl);
that.set('loading', true);
+ store.unloadAll(childEntityType);
+ store.unloadAll('dagProgress');
- this.get('store').unloadAll(childEntityType);
- this.get('store').findQuery(childEntityType, this.getFilterProperties())
- .then(function(entities){
-
- that.set('entities', entities);
-
+ store.findQuery(childEntityType, this.getFilterProperties()).then(function(entities){
var loaders = [];
entities.forEach(function (dag) {
- var applicationId = dag.get('applicationId');
- if (dag.get('status') === 'RUNNING') {
- amInfoFetcher = that.store.find('dagProgress', dag.get('id'), {
- appId: applicationId,
- dagIdx: dag.get('idx')
- }).then(function(dagProgressInfo) {
- dag.set('progress', dagProgressInfo.get('progress'));
- }).catch(function(error) {
- Em.Logger.error('Failed to fetch dagProgress' + error);
+ var appId = dag.get('applicationId');
+ if(appId) {
+ // Pivot attempt selection logic
+ record = store.getById('appDetail', appId);
+ if(record && !App.Helpers.misc.isStatusInUnsuccessful(record.get('appState'))) {
+ store.unloadRecord(record);
+ }
+ fetcher = store.find('appDetail', appId).then(function (app) {
+ dag.set('appDetail', app);
+ if (dag.get('status') === 'RUNNING') {
+ dag.set('status', App.Helpers.misc.getRealStatus(
+ dag.get('status'),
+ app.get('appState'),
+ app.get('finalAppStatus')
+ ));
+ App.Helpers.misc.removeRecord(store, 'tezApp', 'tez_' + appId);
+ }
+ return store.find('tezApp', 'tez_' + appId).then(function (app) {
+ dag.set('tezApp', app);
+ });
});
- loaders.push(amInfoFetcher);
- }
-
- var appDetailLoader = that.store.find('appDetail', applicationId)
- .then(function(app){
- dag.set('appDetail', app);
- var appState = app.get('appState');
- if (appState) {
- dag.set('yarnAppState', appState);
+ loaders.push(fetcher);
+ //Load tezApp details
+ if (dag.get('status') === 'RUNNING') {
+ App.Helpers.misc.removeRecord(store, 'dagProgress', dag.get('id'));
+ amInfoFetcher = store.find('dagProgress', dag.get('id'), {
+ appId: dag.get('applicationId'),
+ dagIdx: dag.get('idx')
+ })
+ .then(function(dagProgressInfo) {
+ dag.set('progress', dagProgressInfo.get('progress'));
+ })
+ .catch(function(error) {
+ Em.Logger.error('Failed to fetch dagProgress' + error);
+ });
+ loaders.push(amInfoFetcher);
}
- dag.set('status', App.Helpers.misc.getRealStatus(dag.get('status'),
- app.get('appState'), app.get('finalAppStatus')));
- });
+ }
});
-
Em.RSVP.allSettled(loaders).then(function(){
+ that.set('entities', entities);
that.set('loading', false);
});
}).catch(function(error){
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-index-controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-index-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-index-controller.js
new file mode 100644
index 0000000..3b11e3a
--- /dev/null
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-index-controller.js
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+App.TezAppIndexController = Em.ObjectController.extend(App.ModelRefreshMixin, {
+
+ needs: "tezApp",
+ controllerName: 'TezAppIndexController',
+
+ load: function () {
+ var tezApp = this.get('model'),
+ store = this.get('store');
+
+ tezApp.reload().then(function (tezApp) {
+ var appId = tezApp.get('appId');
+ if(!appId) return tezApp;
+ App.Helpers.misc.removeRecord(store, 'appDetail', appId);
+ return store.find('appDetail', appId).then(function (appDetails){
+ tezApp.set('appDetail', appDetails);
+ return tezApp;
+ });
+ }).catch(function (error) {
+ Em.Logger.error(error);
+ var err = App.Helpers.misc.formatError(error, defaultErrMsg);
+ var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
+ App.Helpers.ErrorBar.getInstance().show(msg, err.details);
+ });
+ },
+
+ iconStatus: function() {
+ return App.Helpers.misc.getStatusClassForEntity(this.get('model.appDetail.finalAppStatus'));
+ }.property('id', 'appDetail.finalAppStatus'),
+});
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
index 89a99a9..b575d83 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-App.VertexController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
+App.VertexController = Em.ObjectController.extend(App.Helpers.DisplayHelper, App.ModelRefreshMixin, {
controllerName: 'VertexController',
pageTitle: 'Vertex',
@@ -30,6 +30,7 @@ App.VertexController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
if (vertex.get('status') == 'RUNNING') {
var vertexIdx = vertex.get('id').split('_').splice(-1).pop();
+ App.Helpers.misc.removeRecord(this.store, 'vertexProgress', vertexIdx);
var progressLoader = this.store.find('vertexProgress', vertexIdx, {
appId: applicationId,
dagIdx: vertex.get('dagIdx')
@@ -43,6 +44,7 @@ App.VertexController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
loaders.push(progressLoader);
}
+ App.Helpers.misc.removeRecord(that.store, 'appDetail', applicationId);
var appDetailFetcher = that.store.find('appDetail', applicationId).then(function(appDetail) {
var appState = appDetail.get('appState');
if (appState) {
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/vertex_index_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_index_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_index_controller.js
index f175ed1..f104c11 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_index_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_index_controller.js
@@ -16,9 +16,25 @@
* limitations under the License.
*/
-App.VertexIndexController = Em.ObjectController.extend({
+App.VertexIndexController = Em.ObjectController.extend(App.ModelRefreshMixin, {
controllerName: 'VertexIndexController',
+ needs: 'vertex',
+
+ load: function () {
+ var vertex = this.get('controllers.vertex.model'),
+ controller = this.get('controllers.vertex'),
+ t = this;
+ vertex.reload().then(function () {
+ return controller.loadAdditional(vertex);
+ }).catch(function(error){
+ Em.Logger.error(error);
+ var err = App.Helpers.misc.formatError(error, defaultErrMsg);
+ var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
+ App.Helpers.ErrorBar.getInstance().show(msg, err.details);
+ });
+ },
+
//TODO: TEZ-1705 : Create a parent class and move this function there to avoid duplication.
iconStatus: function() {
return App.Helpers.misc.getStatusClassForEntity(this.get('model.status'));
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/vertex_task_attempts_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_task_attempts_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_task_attempts_controller.js
index 5f51de4..c335615 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_task_attempts_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_task_attempts_controller.js
@@ -41,6 +41,23 @@ App.VertexTaskAttemptsController = Em.ObjectController.extend(App.PaginatedConte
this.setFiltersAndLoadEntities(filters);
},
+ load: function () {
+ var vertex = this.get('controllers.vertex.model'),
+ controller = this.get('controllers.vertex'),
+ t = this;
+ vertex.reload().then(function () {
+ return controller.loadAdditional(vertex);
+ }).then(function () {
+ t.resetNavigation();
+ t.loadEntities();
+ }).catch(function(error){
+ Em.Logger.error(error);
+ var err = App.Helpers.misc.formatError(error, defaultErrMsg);
+ var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
+ App.Helpers.ErrorBar.getInstance().show(msg, err.details);
+ });
+ }.observes('count'),
+
actions : {
filterUpdated: function(filterID, value) {
// any validations required goes here.
@@ -53,9 +70,32 @@ App.VertexTaskAttemptsController = Em.ObjectController.extend(App.PaginatedConte
}
},
- defaultColumnConfigs: function() {
- var that = this,
+ updateLoading: function () {
+ var controller = this.get('controllers.vertex'),
+ model = this.get('controllers.vertex.model'),
+ that = this,
vertexStatus = that.get('controllers.vertex.status');
+
+ controller.loadAdditional(model).then(function () {
+ that.get('entities').forEach(function (attempt) {
+
+ var attemptStatus = App.Helpers.misc
+ .getFixedupDisplayStatus(attempt.get('status'));
+ if (attemptStatus == 'RUNNING' &&
+ App.Helpers.misc.isStatusInUnsuccessful(vertexStatus)) {
+ attemptStatus = 'KILLED'
+ }
+ if (attemptStatus != attempt.get('status')) {
+ attempt.set('status', attemptStatus);
+ }
+ });
+
+ that.set('loading', false);
+ });
+ },
+
+ defaultColumnConfigs: function() {
+ var that = this;
return [
{
id: 'taskId',
@@ -127,11 +167,7 @@ App.VertexTaskAttemptsController = Em.ObjectController.extend(App.PaginatedConte
{{view.cellContent.status}}</span>')
}),
getCellContent: function(row) {
- var status = App.Helpers.misc.getFixedupDisplayStatus(row.get('status'));
- if (status == 'RUNNING' &&
- App.Helpers.misc.isStatusInUnsuccessful(vertexStatus)) {
- status = 'KILLED'
- }
+ var status = row.get('status');
return {
status: status,
statusIcon: App.Helpers.misc.getStatusClassForEntity(status)
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/controllers/vertex_tasks_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_tasks_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_tasks_controller.js
index 9b76aee..bba5aa0 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_tasks_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_tasks_controller.js
@@ -41,6 +41,23 @@ App.VertexTasksController = Em.ObjectController.extend(App.PaginatedContentMixin
this.setFiltersAndLoadEntities(filters);
},
+ load: function () {
+ var vertex = this.get('controllers.vertex.model'),
+ controller = this.get('controllers.vertex'),
+ t = this;
+ vertex.reload().then(function () {
+ return controller.loadAdditional(vertex);
+ }).then(function () {
+ t.resetNavigation();
+ t.loadEntities();
+ }).catch(function(error){
+ Em.Logger.error(error);
+ var err = App.Helpers.misc.formatError(error, defaultErrMsg);
+ var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
+ App.Helpers.ErrorBar.getInstance().show(msg, err.details);
+ });
+ }.observes('count'),
+
loadEntities: function() {
var that = this,
store = this.get('store'),
@@ -48,6 +65,7 @@ App.VertexTasksController = Em.ObjectController.extend(App.PaginatedContentMixin
childEntityType = this.get('childEntityType');
var defaultErrMsg = 'Error while loading tasks. could not connect to %@'.fmt(App.env.timelineBaseUrl);
+ that.set('loading', true);
store.unloadAll(childEntityType);
store.findQuery(childEntityType, this.getFilterProperties()).then(function(entities){
var pivotLoaders = [],
@@ -65,6 +83,7 @@ App.VertexTasksController = Em.ObjectController.extend(App.PaginatedContentMixin
var taskAttemptId = task.get('successfulAttemptId') || task.get('attempts.lastObject');
if (!!taskAttemptId) {
// Pivot attempt selection logic
+ App.Helpers.misc.removeRecord(store, 'taskAttempt', taskAttemptId);
fetcher = store.find('taskAttempt', taskAttemptId);
fetcher.then(function (attempt) {
task.set('pivotAttempt', attempt);
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
index 187f800..cebe988 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
@@ -231,6 +231,19 @@ App.Helpers.misc = {
});
},
+ /**
+ * Remove the specific record from store
+ * @param store {DS.Store}
+ * @param type {String}
+ * @param id {String}
+ */
+ removeRecord: function (store, type, id) {
+ var record = store.getById(type, id);
+ if(record) {
+ store.unloadRecord(record);
+ }
+ },
+
dagStatusUIOptions: [
{ label: 'All', id: null },
{ label: 'Submitted', id: 'SUBMITTED' },
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/mixins/model-refresh-mixin.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/mixins/model-refresh-mixin.js b/tez-ui/src/main/webapp/app/scripts/mixins/model-refresh-mixin.js
new file mode 100644
index 0000000..d11744e
--- /dev/null
+++ b/tez-ui/src/main/webapp/app/scripts/mixins/model-refresh-mixin.js
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+App.ModelRefreshMixin = Em.Mixin.create({
+ isRefreshable: true,
+
+ load: function () {
+ var model = this.get('content');
+ if(model && $.isFunction(model.reload)) {
+ model.reload();
+ }
+ },
+
+ actions: {
+ refresh: function () {
+ this.load();
+ }
+ }
+});
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js b/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
index 7ce5f5e..7b08036 100644
--- a/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
+++ b/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
@@ -27,6 +27,8 @@ App.PaginatedContentMixin = Em.Mixin.create({
// The dropdown contents for number of items to show.
countOptions: [5, 10, 25, 50, 100],
+ isRefreshable: true,
+
/* There is currently no efficient way in ATS to get pagination data, so we fake one.
* store the first dag id on a page so that we can navigate back and store the last one
* (not shown on page to get the id where next page starts)
@@ -45,8 +47,9 @@ App.PaginatedContentMixin = Em.Mixin.create({
_paginationFilters: {},
loading: true,
- countUpdated: function() {
- this.loadData();
+ load: function() {
+ this.resetNavigation();
+ this.loadEntities();
}.observes('count'),
sortedContent: function() {
@@ -59,6 +62,10 @@ App.PaginatedContentMixin = Em.Mixin.create({
return sorted.slice(0, this.count);
}.property('entities', 'numEntities'),
+ updateLoading: function () {
+ this.set('loading', false);
+ },
+
loadEntities: function() {
var that = this;
var childEntityType = this.get('childEntityType');
@@ -81,7 +88,7 @@ App.PaginatedContentMixin = Em.Mixin.create({
Em.Logger.error("Exception invoking additional load", error);
}
Em.RSVP.allSettled(loaders).then(function(){
- that.set('loading', false);
+ that.updateLoading();
});
}).catch(function(error){
Em.Logger.error(error);
@@ -93,8 +100,7 @@ App.PaginatedContentMixin = Em.Mixin.create({
setFiltersAndLoadEntities: function(filters) {
this._paginationFilters = filters;
- this.resetNavigation();
- this.loadEntities();
+ this.load();
},
resetNavigation: function() {
@@ -135,6 +141,10 @@ App.PaginatedContentMixin = Em.Mixin.create({
this.loadEntities();
},
+ refresh: function () {
+ this.load();
+ },
+
// goto first page.
navigateFirst: function() {
var firstPageId = this.navIDs.prevIDs[0];
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/models/abstract_entity.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/models/abstract_entity.js b/tez-ui/src/main/webapp/app/scripts/models/abstract_entity.js
index a3e7332..251d680 100644
--- a/tez-ui/src/main/webapp/app/scripts/models/abstract_entity.js
+++ b/tez-ui/src/main/webapp/app/scripts/models/abstract_entity.js
@@ -16,13 +16,24 @@
*/
App.AbstractEntity = DS.Model.extend({
- // type of the entity. should be one of App.EntityType
- entityType: DS.attr('string')
+ // type of the entity. should be one of App.EntityType
+ entityType: DS.attr('string'),
+ timeStamp: null,
+
+ didLoad: function () {
+ this.set('timeStamp', new Date());
+ },
+
+ observeReloading: function () {
+ if(!this.get('isReloading')) {
+ this.didLoad();
+ }
+ }.observes('isReloading')
});
App.EntityType = {
- DAG: 'dag',
- VERTEX: 'vertex',
- TASK: 'task',
+ DAG: 'dag',
+ VERTEX: 'vertex',
+ TASK: 'task',
TASK_ATTEMPT: 'task_attempt',
};
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/scripts/models/dag.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/models/dag.js b/tez-ui/src/main/webapp/app/scripts/models/dag.js
index e050037..9b844f7 100644
--- a/tez-ui/src/main/webapp/app/scripts/models/dag.js
+++ b/tez-ui/src/main/webapp/app/scripts/models/dag.js
@@ -95,7 +95,7 @@ App.Edge = DS.Model.extend({
dag: DS.belongsTo('dag')
});
-App.Vertex = DS.Model.extend({
+App.Vertex = App.AbstractEntity.extend({
name: DS.attr('string'),
vertexIdx: function() {
return this.get('id').split('_').splice(-1).pop();
@@ -257,7 +257,7 @@ App.Vertex = DS.Model.extend({
}.property('duration')
});
-App.Input = DS.Model.extend({
+App.Input = App.AbstractEntity.extend({
entity: DS.attr('string'),
inputName: DS.attr('string'),
@@ -267,7 +267,7 @@ App.Input = DS.Model.extend({
configs: DS.hasMany('kVData', { async: false })
});
-App.Output = DS.Model.extend({
+App.Output = App.AbstractEntity.extend({
entity: DS.attr('string'),
outputName: DS.attr('string'),
@@ -276,7 +276,7 @@ App.Output = DS.Model.extend({
configs: DS.hasMany('kVData', { async: false })
});
-App.AppDetail = DS.Model.extend({
+App.AppDetail = App.AbstractEntity.extend({
attemptId: DS.attr('string'),
user: DS.attr('string'),
@@ -296,7 +296,7 @@ App.AppDetail = DS.Model.extend({
diagnostics: DS.attr('string'),
});
-App.TezApp = DS.Model.extend({
+App.TezApp = App.AbstractEntity.extend({
appId: DS.attr('string'),
entityType: DS.attr('string'),
domain: DS.attr('string'),
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/styles/main.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/main.less b/tez-ui/src/main/webapp/app/styles/main.less
index b29fa6e..be8b228 100644
--- a/tez-ui/src/main/webapp/app/styles/main.less
+++ b/tez-ui/src/main/webapp/app/styles/main.less
@@ -117,6 +117,40 @@ body, html, body > .ember-view {
}
}
+ .table-control {
+ .align-children-left {
+ vertical-align: top;
+ padding-top: 4px;
+ }
+ .align-children-right {
+ margin-top: -8px;
+ }
+ }
+
+ .load-component {
+ margin-bottom: 5px;
+
+ .panel-body {
+ padding-top: 5px;
+ padding-bottom: 5px;
+ }
+
+ .fa-clock-o {
+ .fa-2x;
+
+ position: relative;
+ top: 2px;
+ }
+
+ .align-children-right {
+ vertical-align: top;
+ }
+
+ .margin-small-horizontal {
+ vertical-align: middle;
+ }
+ }
+
.navbar {
margin-bottom: 0px;
}
@@ -410,7 +444,6 @@ div.indent {
.kv-table {
width: 100%;
- margin-top: 10px;
table-layout: fixed;
th, td {
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/styles/shared.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/shared.less b/tez-ui/src/main/webapp/app/styles/shared.less
index 341bb10..617c87f 100644
--- a/tez-ui/src/main/webapp/app/styles/shared.less
+++ b/tez-ui/src/main/webapp/app/styles/shared.less
@@ -33,6 +33,10 @@
display: none !important;
}
+.no-visible {
+ visibility: hidden !important;
+}
+
.no-pointer {
pointer-events: none;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/common/counters.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/common/counters.hbs b/tez-ui/src/main/webapp/app/templates/common/counters.hbs
index e41e8f4..30419e1 100644
--- a/tez-ui/src/main/webapp/app/templates/common/counters.hbs
+++ b/tez-ui/src/main/webapp/app/templates/common/counters.hbs
@@ -16,6 +16,14 @@
* limitations under the License.
}}
-<div class='table-container'>
- {{counter-table-component data=counterGroups}}
+{{load-time-component
+ isRefreshable=isRefreshable
+ time=timeStamp
+ refresh='refresh'
+}}
+<div class='table-container margin-small-vertical'>
+ {{counter-table-component
+ data=counterGroups
+ timeStamp=timeStamp
+ }}
</div>
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/common/swimlanes.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/common/swimlanes.hbs b/tez-ui/src/main/webapp/app/templates/common/swimlanes.hbs
index 2745e08..4d051b8 100644
--- a/tez-ui/src/main/webapp/app/templates/common/swimlanes.hbs
+++ b/tez-ui/src/main/webapp/app/templates/common/swimlanes.hbs
@@ -16,6 +16,11 @@
* limitations under the License.
}}
+{{load-time-component
+ isRefreshable=isRefreshable
+ time=controller.model.content.0.timeStamp
+ refresh='refresh'
+}}
<div id="swimlane">
{{#view App.SwimlanesView contentBinding="controller.model.content"}}
{{/view}}
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/common/table-with-spinner.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/common/table-with-spinner.hbs b/tez-ui/src/main/webapp/app/templates/common/table-with-spinner.hbs
index e1c154e..88b5d63 100644
--- a/tez-ui/src/main/webapp/app/templates/common/table-with-spinner.hbs
+++ b/tez-ui/src/main/webapp/app/templates/common/table-with-spinner.hbs
@@ -17,7 +17,12 @@
}}
{{#unless loading}}
- <div class='margin-small align-children-right'>
+ {{load-time-component
+ isRefreshable=isRefreshable
+ time=sortedContent.0.timeStamp
+ refresh='refresh'
+ }}
+ <div class='align-children-right'>
{{partial 'partials/table-controls'}}
</div>
{{partial 'partials/table'}}
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/components/counter-table.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/counter-table.hbs b/tez-ui/src/main/webapp/app/templates/components/counter-table.hbs
index e8d12b1..e9070f1 100644
--- a/tez-ui/src/main/webapp/app/templates/components/counter-table.hbs
+++ b/tez-ui/src/main/webapp/app/templates/components/counter-table.hbs
@@ -29,12 +29,12 @@
<tbody>
{{#each counterGroup in filteredData}}
<tr class='group-header'>
- <td colspan='2'>{{unbound counterGroup.displayName}}</td>
+ <td colspan='2'>{{counterGroup.displayName}}</td>
</tr>
{{#each counter in counterGroup.counters}}
<tr>
- <td>{{unbound counter.displayName}}</td>
- <td>{{unbound formatNumThousands counter.value}}</td>
+ <td>{{counter.displayName}}</td>
+ <td>{{formatNumThousands counter.value}}</td>
</tr>
{{/each}}
{{/each}}
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/components/load-time.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/load-time.hbs b/tez-ui/src/main/webapp/app/templates/components/load-time.hbs
new file mode 100644
index 0000000..3f6881f
--- /dev/null
+++ b/tez-ui/src/main/webapp/app/templates/components/load-time.hbs
@@ -0,0 +1,37 @@
+{{!
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+}}
+
+<div class="panel panel-default load-component">
+ <div class="panel-body">
+ <div class="horizontal-half">
+ <i class='fa fa-clock-o'></i>
+ </div><div class="horizontal-half align-children-right">
+ <span class="margin-small-horizontal">
+ {{#if displayTime}}
+ Last refreshed at <b>{{displayTime}}</b>
+ {{else}}
+ Load time not available!
+ {{/if}}
+ </span>
+ <button type="button" {{bind-attr class=":btn :btn-success :btn-sm isRefreshable::no-display"}} {{action 'refresh'}}>
+ <i class='fa fa-refresh'></i>
+ Refresh
+ </button>
+ </div>
+ </div>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/dag/index.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/dag/index.hbs b/tez-ui/src/main/webapp/app/templates/dag/index.hbs
index 7e9dbcb..bc6b1d3 100644
--- a/tez-ui/src/main/webapp/app/templates/dag/index.hbs
+++ b/tez-ui/src/main/webapp/app/templates/dag/index.hbs
@@ -16,6 +16,13 @@
* limitations under the License.
}}
+<div class="margin-small-vertical">
+ {{load-time-component
+ isRefreshable=isRefreshable
+ time=timeStamp
+ refresh='refresh'
+ }}
+</div>
<div class='type-table fill-full margin-small-horizontal'>
<div class='align-left'>
<table class='detail-list'>
@@ -23,8 +30,7 @@
<tr>
<td>{{t 'common.status'}}</td>
<td>
- {{status}}
- <i {{bind-attr class=':task-status taskIconStatus'}}></i>
+ <i {{bind-attr class=':task-status taskIconStatus'}}></i> {{status}}
{{#if progressStr}} {{bs-badge content=progressStr}}{{/if}}
</td>
<td>
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/dag/view.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/dag/view.hbs b/tez-ui/src/main/webapp/app/templates/dag/view.hbs
index 27ee267..699d0c8 100644
--- a/tez-ui/src/main/webapp/app/templates/dag/view.hbs
+++ b/tez-ui/src/main/webapp/app/templates/dag/view.hbs
@@ -16,6 +16,13 @@
* limitations under the License.
}}
+<div {{bind-attr class=":margin-small-vertical loading:no-visible"}}>
+ {{load-time-component
+ isRefreshable=isRefreshable
+ time=entities.content.0.timeStamp
+ refresh='refresh'
+ }}
+</div>
<div class="dag-view-component-container">
{{#unless loading}}
{{dag-view-component
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/partials/configs.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/partials/configs.hbs b/tez-ui/src/main/webapp/app/templates/partials/configs.hbs
index 0e00ca5..0370890 100644
--- a/tez-ui/src/main/webapp/app/templates/partials/configs.hbs
+++ b/tez-ui/src/main/webapp/app/templates/partials/configs.hbs
@@ -17,7 +17,12 @@
}}
{{#if configs}}
- <div class='table-container'>
+ {{load-time-component
+ isRefreshable=isRefreshable
+ time=timeStamp
+ refresh='refresh'
+ }}
+ <div class='table-container margin-small-vertical'>
{{kv-table-component data=configs filterExp="^tez."}}
</div>
{{else}}
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/partials/table-controls.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/partials/table-controls.hbs b/tez-ui/src/main/webapp/app/templates/partials/table-controls.hbs
index 1b5745f..5cc6840 100644
--- a/tez-ui/src/main/webapp/app/templates/partials/table-controls.hbs
+++ b/tez-ui/src/main/webapp/app/templates/partials/table-controls.hbs
@@ -16,7 +16,7 @@
* limitations under the License.
}}
-<div class='margin-small'>
+<div class='margin-small-horizontal table-control'>
<div class="horizontal-half align-children-left">
Number of rows
{{view Ember.Select
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/task/index.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/task/index.hbs b/tez-ui/src/main/webapp/app/templates/task/index.hbs
index 6826e7b..833e624 100644
--- a/tez-ui/src/main/webapp/app/templates/task/index.hbs
+++ b/tez-ui/src/main/webapp/app/templates/task/index.hbs
@@ -16,6 +16,13 @@
* limitations under the License.
}}
+<div class="margin-small-vertical">
+ {{load-time-component
+ isRefreshable=isRefreshable
+ time=timeStamp
+ refresh='refresh'
+ }}
+</div>
<div class='type-table fill-full margin-small-horizontal'>
<div class='align-left'>
<table class='detail-list'>
@@ -23,8 +30,7 @@
<tr>
<td>{{t 'common.status'}}</td>
<td>
- {{taskStatus}}
- <i {{bind-attr class=':task-status taskIconStatus'}}></i>
+ <i {{bind-attr class=':task-status taskIconStatus'}}></i> {{taskStatus}}
</td>
</tr>
</tbody>
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/taskAttempt/index.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/taskAttempt/index.hbs b/tez-ui/src/main/webapp/app/templates/taskAttempt/index.hbs
index 738b636..23b96b4 100644
--- a/tez-ui/src/main/webapp/app/templates/taskAttempt/index.hbs
+++ b/tez-ui/src/main/webapp/app/templates/taskAttempt/index.hbs
@@ -16,6 +16,13 @@
* limitations under the License.
}}
+<div class="margin-small-vertical">
+ {{load-time-component
+ isRefreshable=isRefreshable
+ time=timeStamp
+ refresh='refresh'
+ }}
+</div>
<div class='type-table fill-full margin-small-horizontal'>
<div class='align-left'>
<table class='detail-list'>
@@ -23,8 +30,7 @@
<tr>
<td>{{t 'common.status'}}</td>
<td>
- {{taskAttemptStatus}}
- <i {{bind-attr class=':task-status taskAttemptIconStatus'}}></i>
+ <i {{bind-attr class=':task-status taskAttemptIconStatus'}}></i> {{taskAttemptStatus}}
</td>
</tr>
<tr>
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/tez-app/index.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/tez-app/index.hbs b/tez-ui/src/main/webapp/app/templates/tez-app/index.hbs
index 07777dd..bedb8f2 100644
--- a/tez-ui/src/main/webapp/app/templates/tez-app/index.hbs
+++ b/tez-ui/src/main/webapp/app/templates/tez-app/index.hbs
@@ -16,6 +16,13 @@
* limitations under the License.
}}
+<div class="margin-small-vertical">
+ {{load-time-component
+ isRefreshable=isRefreshable
+ time=timeStamp
+ refresh='refresh'
+ }}
+</div>
<div class='type-table fill-full margin-small-horizontal'>
<div class='align-left'>
<table class='detail-list'>
@@ -48,7 +55,9 @@
</tr>
<tr>
<td>Final App Status</td>
- <td>{{appDetail.finalAppStatus}}</td>
+ <td>
+ <i {{bind-attr class=':task-status iconStatus'}}></i> {{appDetail.finalAppStatus}}
+ </td>
</tr>
</tbody>
</table>
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/vertex/additionals.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/vertex/additionals.hbs b/tez-ui/src/main/webapp/app/templates/vertex/additionals.hbs
index bb85eb3..f6ee573 100644
--- a/tez-ui/src/main/webapp/app/templates/vertex/additionals.hbs
+++ b/tez-ui/src/main/webapp/app/templates/vertex/additionals.hbs
@@ -16,6 +16,13 @@
* limitations under the License.
}}
+<div class="margin-small-vertical">
+ {{load-time-component
+ isRefreshable=isRefreshable
+ time=timeStamp
+ refresh='refresh'
+ }}
+</div>
{{#unless loading}}
<h3>Sources</h3>
{{#if inputsAvailable}}
http://git-wip-us.apache.org/repos/asf/tez/blob/dd44bdec/tez-ui/src/main/webapp/app/templates/vertex/index.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/vertex/index.hbs b/tez-ui/src/main/webapp/app/templates/vertex/index.hbs
index bc2e7ef..557c644 100644
--- a/tez-ui/src/main/webapp/app/templates/vertex/index.hbs
+++ b/tez-ui/src/main/webapp/app/templates/vertex/index.hbs
@@ -16,6 +16,13 @@
* limitations under the License.
}}
+<div class="margin-small-vertical">
+ {{load-time-component
+ isRefreshable=isRefreshable
+ time=timeStamp
+ refresh='refresh'
+ }}
+</div>
<div class='type-table margin-small-horizontal fill-full'>
<div class="horizontal-half align-children-left">
[20/23] tez git commit: TEZ-2006. Task communication plane needs to
be pluggable. (sseth)
Posted by ss...@apache.org.
TEZ-2006. Task communication plane needs to be pluggable. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a7a760fe
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a7a760fe
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a7a760fe
Branch: refs/heads/TEZ-2003
Commit: a7a760fec5d5675d1c292bb156b9189bccd0f2a8
Parents: ee95457
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Feb 12 11:25:45 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Feb 18 14:40:12 2015 -0800
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 1 +
.../apache/tez/dag/api/TaskCommunicator.java | 54 ++
.../tez/dag/api/TaskCommunicatorContext.java | 48 ++
.../tez/dag/api/TaskHeartbeatRequest.java | 63 +++
.../tez/dag/api/TaskHeartbeatResponse.java | 39 ++
.../java/org/apache/tez/dag/app/AppContext.java | 3 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 5 +
.../dag/app/TaskAttemptListenerImpTezDag.java | 501 +++++++------------
.../tez/dag/app/TezTaskCommunicatorImpl.java | 474 ++++++++++++++++++
.../app/launcher/LocalContainerLauncher.java | 10 +-
.../tez/dag/app/rm/container/AMContainer.java | 3 +-
.../rm/container/AMContainerEventAssignTA.java | 2 +
.../dag/app/rm/container/AMContainerImpl.java | 1 +
.../apache/tez/dag/app/MockDAGAppMaster.java | 7 +-
.../app/TestTaskAttemptListenerImplTezDag.java | 81 +--
15 files changed, 945 insertions(+), 347 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 1822fcb..d7e4be5 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -1,4 +1,5 @@
ALL CHANGES:
TEZ-2019. Temporarily allow the scheduler and launcher to be specified via configuration.
+ TEZ-2006. Task communication plane needs to be pluggable.
INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
new file mode 100644
index 0000000..97f9c16
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.net.InetSocketAddress;
+import java.util.Map;
+
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+
+// TODO TEZ-2003 Move this into the tez-api module
+public abstract class TaskCommunicator extends AbstractService {
+ public TaskCommunicator(String name) {
+ super(name);
+ }
+
+ // TODO TEZ-2003 Ideally, don't expose YARN containerId; instead expose a Tez specific construct.
+ // TODO When talking to an external service, this plugin implementer may need access to a host:port
+ public abstract void registerRunningContainer(ContainerId containerId, String hostname, int port);
+
+ // TODO TEZ-2003 Ideally, don't expose YARN containerId; instead expose a Tez specific construct.
+ public abstract void registerContainerEnd(ContainerId containerId);
+
+ // TODO TEZ-2003 TaskSpec breakup into a clean interface
+ // TODO TEZ-2003 Add support for priority
+ public abstract void registerRunningTaskAttempt(ContainerId containerId, TaskSpec taskSpec,
+ Map<String, LocalResource> additionalResources,
+ Credentials credentials,
+ boolean credentialsChanged);
+
+ // TODO TEZ-2003 Remove reference to TaskAttemptID
+ public abstract void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID);
+
+ // TODO TEZ-2003 This doesn't necessarily belong here. A server may not start within the AM.
+ public abstract InetSocketAddress getAddress();
+
+ // TODO Eventually. Add methods here to support preemption of tasks.
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
new file mode 100644
index 0000000..9b2d889
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskCommunicatorContext.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+
+
+// Do not make calls into this from within a held lock.
+
+// TODO TEZ-2003 Move this into the tez-api module
+public interface TaskCommunicatorContext {
+
+ // TODO TEZ-2003 Add signalling back into this to indicate errors - e.g. Container unregsitered, task no longer running, etc.
+
+ // TODO TEZ-2003 Maybe add book-keeping as a helper library, instead of each impl tracking container to task etc.
+
+ ApplicationAttemptId getApplicationAttemptId();
+ Credentials getCredentials();
+
+ // TODO TEZ-2003 Move to vertex, taskIndex, version
+ boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException;
+
+ TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request) throws IOException, TezException;
+
+ boolean isKnownContainer(ContainerId containerId);
+
+ // TODO TEZ-2003 Move to vertex, taskIndex, version
+ void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId);
+
+ // TODO Eventually Add methods to report availability stats to the scheduler.
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatRequest.java
new file mode 100644
index 0000000..f6bc8f0
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatRequest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.util.List;
+
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TezEvent;
+
+// TODO TEZ-2003 Move this into the tez-api module
+public class TaskHeartbeatRequest {
+
+ // TODO TEZ-2003 Ideally containerIdentifier should not be part of the request.
+ // Replace with a task lookup - vertex name + task index
+ private final String containerIdentifier;
+ // TODO TEZ-2003 Get rid of the task attemptId reference if possible
+ private final TezTaskAttemptID taskAttemptId;
+ private final List<TezEvent> events;
+ private final int startIndex;
+ private final int maxEvents;
+
+
+ public TaskHeartbeatRequest(String containerIdentifier, TezTaskAttemptID taskAttemptId, List<TezEvent> events, int startIndex,
+ int maxEvents) {
+ this.containerIdentifier = containerIdentifier;
+ this.taskAttemptId = taskAttemptId;
+ this.events = events;
+ this.startIndex = startIndex;
+ this.maxEvents = maxEvents;
+ }
+
+ public String getContainerIdentifier() {
+ return containerIdentifier;
+ }
+
+ public TezTaskAttemptID getTaskAttemptId() {
+ return taskAttemptId;
+ }
+
+ public List<TezEvent> getEvents() {
+ return events;
+ }
+
+ public int getStartIndex() {
+ return startIndex;
+ }
+
+ public int getMaxEvents() {
+ return maxEvents;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatResponse.java b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatResponse.java
new file mode 100644
index 0000000..c82a743
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/TaskHeartbeatResponse.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.util.List;
+
+import org.apache.tez.runtime.api.impl.TezEvent;
+
+// TODO TEZ-2003 Move this into the tez-api module
+public class TaskHeartbeatResponse {
+
+ private final boolean shouldDie;
+ private List<TezEvent> events;
+
+ public TaskHeartbeatResponse(boolean shouldDie, List<TezEvent> events) {
+ this.shouldDie = shouldDie;
+ this.events = events;
+ }
+
+ public boolean isShouldDie() {
+ return shouldDie;
+ }
+
+ public List<TezEvent> getEvents() {
+ return events;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index 5564809..f22b17a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -102,4 +103,6 @@ public interface AppContext {
String[] getLocalDirs();
String getAMUser();
+
+ Credentials getAppCredentials();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 9f523ac..c980a3d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -1385,6 +1385,11 @@ public class DAGAppMaster extends AbstractService {
}
@Override
+ public Credentials getAppCredentials() {
+ return amCredentials;
+ }
+
+ @Override
public Map<ApplicationAccessType, String> getApplicationACLs() {
if (getServiceState() != STATE.STARTED) {
throw new TezUncheckedException(
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 4cb5e99..08b50ba 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -18,228 +18,209 @@
package org.apache.tez.dag.app;
import java.io.IOException;
-import java.net.InetAddress;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.net.InetSocketAddress;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TaskCommunicator;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.api.TaskHeartbeatResponse;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.common.ContainerContext;
-import org.apache.tez.common.ContainerTask;
-import org.apache.tez.common.TezConverterUtils;
-import org.apache.tez.common.TezLocalResource;
-import org.apache.tez.common.TezTaskUmbilicalProtocol;
-import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TaskHeartbeatRequest;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
+import org.apache.tez.dag.app.rm.TaskSchedulerService;
import org.apache.tez.dag.app.rm.container.AMContainerTask;
-import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.runtime.api.impl.TezEvent;
-import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
-import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
import org.apache.tez.common.security.JobTokenSecretManager;
-import com.google.common.collect.Maps;
@SuppressWarnings("unchecked")
+@InterfaceAudience.Private
public class TaskAttemptListenerImpTezDag extends AbstractService implements
- TezTaskUmbilicalProtocol, TaskAttemptListener {
-
- private static final ContainerTask TASK_FOR_INVALID_JVM = new ContainerTask(
- null, true, null, null, false);
+ TaskAttemptListener, TaskCommunicatorContext {
private static final Log LOG = LogFactory
.getLog(TaskAttemptListenerImpTezDag.class);
private final AppContext context;
+ private TaskCommunicator taskCommunicator;
protected final TaskHeartbeatHandler taskHeartbeatHandler;
protected final ContainerHeartbeatHandler containerHeartbeatHandler;
- private final JobTokenSecretManager jobTokenSecretManager;
- private InetSocketAddress address;
- private Server server;
-
- static class ContainerInfo {
- ContainerInfo() {
- this.lastReponse = null;
- this.lastRequestId = 0;
- this.amContainerTask = null;
- this.taskPulled = false;
+
+ private final TaskHeartbeatResponse RESPONSE_SHOULD_DIE = new TaskHeartbeatResponse(true, null);
+
+ private final ConcurrentMap<TezTaskAttemptID, ContainerId> registeredAttempts =
+ new ConcurrentHashMap<TezTaskAttemptID, ContainerId>();
+ private final ConcurrentMap<ContainerId, ContainerInfo> registeredContainers =
+ new ConcurrentHashMap<ContainerId, ContainerInfo>();
+
+ // Defined primarily to work around ConcurrentMaps not accepting null values
+ private static final class ContainerInfo {
+ TezTaskAttemptID taskAttemptId;
+ ContainerInfo(TezTaskAttemptID taskAttemptId) {
+ this.taskAttemptId = taskAttemptId;
}
- long lastRequestId;
- TezHeartbeatResponse lastReponse;
- AMContainerTask amContainerTask;
- boolean taskPulled;
}
- private ConcurrentMap<TezTaskAttemptID, ContainerId> attemptToInfoMap =
- new ConcurrentHashMap<TezTaskAttemptID, ContainerId>();
+ private static final ContainerInfo NULL_CONTAINER_INFO = new ContainerInfo(null);
- private ConcurrentHashMap<ContainerId, ContainerInfo> registeredContainers =
- new ConcurrentHashMap<ContainerId, ContainerInfo>();
public TaskAttemptListenerImpTezDag(AppContext context,
- TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
- JobTokenSecretManager jobTokenSecretManager) {
+ TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
+ // TODO TEZ-2003 pre-merge. Remove reference to JobTokenSecretManager.
+ JobTokenSecretManager jobTokenSecretManager) {
super(TaskAttemptListenerImpTezDag.class.getName());
this.context = context;
- this.jobTokenSecretManager = jobTokenSecretManager;
this.taskHeartbeatHandler = thh;
this.containerHeartbeatHandler = chh;
+ this.taskCommunicator = new TezTaskCommunicatorImpl(this);
}
@Override
- public void serviceStart() {
- startRpcServer();
- }
-
- protected void startRpcServer() {
- Configuration conf = getConfig();
- if (!conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT)) {
- try {
- server = new RPC.Builder(conf)
- .setProtocol(TezTaskUmbilicalProtocol.class)
- .setBindAddress("0.0.0.0")
- .setPort(0)
- .setInstance(this)
- .setNumHandlers(
- conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
- TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT))
- .setSecretManager(jobTokenSecretManager).build();
-
- // Enable service authorization?
- if (conf.getBoolean(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
- false)) {
- refreshServiceAcls(conf, new TezAMPolicyProvider());
- }
-
- server.start();
- this.address = NetUtils.getConnectAddress(server);
- } catch (IOException e) {
- throw new TezUncheckedException(e);
- }
+ public void serviceInit(Configuration conf) {
+ String taskCommClassName = conf.get(TezConfiguration.TEZ_AM_TASK_COMMUNICATOR_CLASS);
+ if (taskCommClassName == null) {
+ LOG.info("Using Default Task Communicator");
+ this.taskCommunicator = new TezTaskCommunicatorImpl(this);
} else {
+ LOG.info("Using TaskCommunicator: " + taskCommClassName);
+ Class<? extends TaskCommunicator> taskCommClazz = (Class<? extends TaskCommunicator>) ReflectionUtils
+ .getClazz(taskCommClassName);
try {
- this.address = new InetSocketAddress(InetAddress.getLocalHost(), 0);
- } catch (UnknownHostException e) {
+ Constructor<? extends TaskCommunicator> ctor = taskCommClazz.getConstructor(TaskCommunicatorContext.class);
+ ctor.setAccessible(true);
+ this.taskCommunicator = ctor.newInstance(this);
+ } catch (NoSuchMethodException e) {
+ throw new TezUncheckedException(e);
+ } catch (InvocationTargetException e) {
+ throw new TezUncheckedException(e);
+ } catch (InstantiationException e) {
+ throw new TezUncheckedException(e);
+ } catch (IllegalAccessException e) {
throw new TezUncheckedException(e);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Not starting TaskAttemptListener RPC in LocalMode");
}
}
}
- void refreshServiceAcls(Configuration configuration,
- PolicyProvider policyProvider) {
- this.server.refreshServiceAcl(configuration, policyProvider);
+ @Override
+ public void serviceStart() {
+ taskCommunicator.init(getConfig());
+ taskCommunicator.start();
}
@Override
public void serviceStop() {
- stopRpcServer();
- }
-
- protected void stopRpcServer() {
- if (server != null) {
- server.stop();
+ if (taskCommunicator != null) {
+ taskCommunicator.stop();
+ taskCommunicator = null;
}
}
- public InetSocketAddress getAddress() {
- return address;
- }
-
@Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return versionID;
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return context.getApplicationAttemptId();
}
@Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- return ProtocolSignature.getProtocolSignature(this, protocol,
- clientVersion, clientMethodsHash);
+ public Credentials getCredentials() {
+ return context.getAppCredentials();
}
@Override
- public ContainerTask getTask(ContainerContext containerContext)
- throws IOException {
+ public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request)
+ throws IOException, TezException {
+ ContainerId containerId = ConverterUtils.toContainerId(request
+ .getContainerIdentifier());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received heartbeat from container"
+ + ", request=" + request);
+ }
- ContainerTask task = null;
+ if (!registeredContainers.containsKey(containerId)) {
+ LOG.warn("Received task heartbeat from unknown container with id: " + containerId +
+ ", asking it to die");
+ return RESPONSE_SHOULD_DIE;
+ }
- if (containerContext == null || containerContext.getContainerIdentifier() == null) {
- LOG.info("Invalid task request with an empty containerContext or containerId");
- task = TASK_FOR_INVALID_JVM;
- } else {
- ContainerId containerId = ConverterUtils.toContainerId(containerContext
- .getContainerIdentifier());
+ // A heartbeat can come in anytime. The AM may have made a decision to kill a running task/container
+ // meanwhile. If the decision is processed through the pipeline before the heartbeat is processed,
+ // the heartbeat will be dropped. Otherwise the heartbeat will be processed - and the system
+ // know how to handle this - via FailedInputEvents for example (relevant only if the heartbeat has events).
+ // So - avoiding synchronization.
+
+ pingContainerHeartbeatHandler(containerId);
+ List<TezEvent> outEvents = null;
+ TezTaskAttemptID taskAttemptID = request.getTaskAttemptId();
+ if (taskAttemptID != null) {
+ ContainerId containerIdFromMap = registeredAttempts.get(taskAttemptID);
+ if (containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
+ // This can happen when a task heartbeats. Meanwhile the container is unregistered.
+ // The information will eventually make it through to the plugin via a corresponding unregister.
+ // There's a race in that case between the unregister making it through, and this method returning.
+ // TODO TEZ-2003. An exception back is likely a better approach than sending a shouldDie = true,
+ // so that the plugin can handle the scenario. Alternately augment the response with error codes.
+ // Error codes would be better than exceptions.
+ LOG.info("Attempt: " + taskAttemptID + " is not recognized for heartbeats");
+ return RESPONSE_SHOULD_DIE;
+ }
+
+ List<TezEvent> inEvents = request.getEvents();
if (LOG.isDebugEnabled()) {
- LOG.debug("Container with id: " + containerId + " asked for a task");
+ LOG.debug("Ping from " + taskAttemptID.toString() +
+ " events: " + (inEvents != null ? inEvents.size() : -1));
}
- if (!registeredContainers.containsKey(containerId)) {
- if(context.getAllContainers().get(containerId) == null) {
- LOG.info("Container with id: " + containerId
- + " is invalid and will be killed");
- } else {
- LOG.info("Container with id: " + containerId
- + " is valid, but no longer registered, and will be killed");
- }
- task = TASK_FOR_INVALID_JVM;
- } else {
- pingContainerHeartbeatHandler(containerId);
- task = getContainerTask(containerId);
- if (task == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No task current assigned to Container with id: " + containerId);
- }
- } else if (task == TASK_FOR_INVALID_JVM) {
- LOG.info("Container with id: " + containerId
- + " is valid, but no longer registered, and will be killed. Race condition.");
- } else {
- context.getEventHandler().handle(
- new TaskAttemptEventStartedRemotely(task.getTaskSpec()
- .getTaskAttemptID(), containerId, context
- .getApplicationACLs()));
- LOG.info("Container with id: " + containerId + " given task: "
- + task.getTaskSpec().getTaskAttemptID());
- }
+ if (inEvents != null && !inEvents.isEmpty()) {
+ TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
+ context.getEventHandler().handle(
+ new VertexEventRouteEvent(vertexId, inEvents));
}
+ taskHeartbeatHandler.pinged(taskAttemptID);
+ outEvents = context
+ .getCurrentDAG()
+ .getVertex(taskAttemptID.getTaskID().getVertexID())
+ .getTask(taskAttemptID.getTaskID())
+ .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(),
+ request.getMaxEvents());
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("getTask returning task: " + task);
- }
- return task;
+ return new TaskHeartbeatResponse(false, outEvents);
+ }
+
+ @Override
+ public boolean isKnownContainer(ContainerId containerId) {
+ return context.getAllContainers().get(containerId) != null;
+ }
+
+ @Override
+ public void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId) {
+ context.getEventHandler().handle(new TaskAttemptEventStartedRemotely(taskAttemptID, containerId, null));
+ pingContainerHeartbeatHandler(containerId);
}
/**
* Child checking whether it can commit.
- *
+ * <p/>
* <br/>
* Repeatedly polls the ApplicationMaster whether it
* {@link Task#canCommit(TezTaskAttemptID)} This is * a legacy from the
@@ -262,72 +243,91 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
@Override
- public void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
- ContainerId containerId = attemptToInfoMap.get(attemptId);
- if(containerId == null) {
- LOG.warn("Unregister task attempt: " + attemptId + " from unknown container");
- return;
- }
- ContainerInfo containerInfo = registeredContainers.get(containerId);
- if(containerInfo == null) {
- LOG.warn("Unregister task attempt: " + attemptId +
- " from non-registered container: " + containerId);
- return;
- }
- synchronized (containerInfo) {
- containerInfo.amContainerTask = null;
- attemptToInfoMap.remove(attemptId);
- }
-
+ public InetSocketAddress getAddress() {
+ return taskCommunicator.getAddress();
}
+ // The TaskAttemptListener register / unregister methods in this class are not thread safe.
+ // The Tez framework should not invoke these methods from multiple threads.
@Override
public void registerRunningContainer(ContainerId containerId) {
if (LOG.isDebugEnabled()) {
- LOG.debug("ContainerId: " + containerId
- + " registered with TaskAttemptListener");
+ LOG.debug("ContainerId: " + containerId + " registered with TaskAttemptListener");
}
- ContainerInfo oldInfo = registeredContainers.put(containerId, new ContainerInfo());
- if(oldInfo != null) {
+ ContainerInfo oldInfo = registeredContainers.put(containerId, NULL_CONTAINER_INFO);
+ if (oldInfo != null) {
throw new TezUncheckedException(
"Multiple registrations for containerId: " + containerId);
}
+ NodeId nodeId = context.getAllContainers().get(containerId).getContainer().getNodeId();
+ taskCommunicator.registerRunningContainer(containerId, nodeId.getHost(), nodeId.getPort());
+ }
+
+ @Override
+ public void unregisterRunningContainer(ContainerId containerId) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unregistering Container from TaskAttemptListener: " + containerId);
+ }
+ ContainerInfo containerInfo = registeredContainers.remove(containerId);
+ if (containerInfo.taskAttemptId != null) {
+ registeredAttempts.remove(containerInfo.taskAttemptId);
+ }
+ taskCommunicator.registerContainerEnd(containerId);
}
@Override
public void registerTaskAttempt(AMContainerTask amContainerTask,
- ContainerId containerId) {
+ ContainerId containerId) {
ContainerInfo containerInfo = registeredContainers.get(containerId);
- if(containerInfo == null) {
+ if (containerInfo == null) {
throw new TezUncheckedException("Registering task attempt: "
+ amContainerTask.getTask().getTaskAttemptID() + " to unknown container: " + containerId);
}
- synchronized (containerInfo) {
- if(containerInfo.amContainerTask != null) {
- throw new TezUncheckedException("Registering task attempt: "
- + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
- + " with existing assignment to: " + containerInfo.amContainerTask.getTask().getTaskAttemptID());
- }
- containerInfo.amContainerTask = amContainerTask;
- containerInfo.taskPulled = false;
-
- ContainerId containerIdFromMap =
- attemptToInfoMap.put(amContainerTask.getTask().getTaskAttemptID(), containerId);
- if(containerIdFromMap != null) {
- throw new TezUncheckedException("Registering task attempt: "
- + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
- + " when already assigned to: " + containerIdFromMap);
- }
+ if (containerInfo.taskAttemptId != null) {
+ throw new TezUncheckedException("Registering task attempt: "
+ + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+ + " with existing assignment to: " +
+ containerInfo.taskAttemptId);
+ }
+
+ if (containerInfo.taskAttemptId != null) {
+ throw new TezUncheckedException("Registering task attempt: "
+ + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+ + " with existing assignment to: " +
+ containerInfo.taskAttemptId);
}
+
+ // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
+ registeredContainers.put(containerId, new ContainerInfo(amContainerTask.getTask().getTaskAttemptID()));
+
+ ContainerId containerIdFromMap = registeredAttempts.put(
+ amContainerTask.getTask().getTaskAttemptID(), containerId);
+ if (containerIdFromMap != null) {
+ throw new TezUncheckedException("Registering task attempt: "
+ + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+ + " when already assigned to: " + containerIdFromMap);
+ }
+ taskCommunicator.registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
+ amContainerTask.getAdditionalResources(), amContainerTask.getCredentials(),
+ amContainerTask.haveCredentialsChanged());
}
@Override
- public void unregisterRunningContainer(ContainerId containerId) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Unregistering Container from TaskAttemptListener: "
- + containerId);
+ public void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
+ ContainerId containerId = registeredAttempts.remove(attemptId);
+ if (containerId == null) {
+ LOG.warn("Unregister task attempt: " + attemptId + " from unknown container");
+ return;
+ }
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ if (containerInfo == null) {
+ LOG.warn("Unregister task attempt: " + attemptId +
+ " from non-registered container: " + containerId);
+ return;
}
- registeredContainers.remove(containerId);
+ // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
+ registeredContainers.put(containerId, NULL_CONTAINER_INFO);
+ taskCommunicator.unregisterRunningTaskAttempt(attemptId);
}
private void pingContainerHeartbeatHandler(ContainerId containerId) {
@@ -335,7 +335,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
private void pingContainerHeartbeatHandler(TezTaskAttemptID taskAttemptId) {
- ContainerId containerId = attemptToInfoMap.get(taskAttemptId);
+ ContainerId containerId = registeredAttempts.get(taskAttemptId);
if (containerId != null) {
containerHeartbeatHandler.pinged(containerId);
} else {
@@ -344,124 +344,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
}
- @Override
- public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request)
- throws IOException, TezException {
- ContainerId containerId = ConverterUtils.toContainerId(request
- .getContainerIdentifier());
- long requestId = request.getRequestId();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Received heartbeat from container"
- + ", request=" + request);
- }
-
- ContainerInfo containerInfo = registeredContainers.get(containerId);
- if(containerInfo == null) {
- LOG.warn("Received task heartbeat from unknown container with id: " + containerId +
- ", asking it to die");
- TezHeartbeatResponse response = new TezHeartbeatResponse();
- response.setLastRequestId(requestId);
- response.setShouldDie();
- return response;
- }
-
- synchronized (containerInfo) {
- pingContainerHeartbeatHandler(containerId);
-
- if(containerInfo.lastRequestId == requestId) {
- LOG.warn("Old sequenceId received: " + requestId
- + ", Re-sending last response to client");
- return containerInfo.lastReponse;
- }
-
- TezHeartbeatResponse response = new TezHeartbeatResponse();
- response.setLastRequestId(requestId);
-
- TezTaskAttemptID taskAttemptID = request.getCurrentTaskAttemptID();
- if (taskAttemptID != null) {
- ContainerId containerIdFromMap = attemptToInfoMap.get(taskAttemptID);
- if(containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
- throw new TezException("Attempt " + taskAttemptID
- + " is not recognized for heartbeat");
- }
-
- if(containerInfo.lastRequestId+1 != requestId) {
- throw new TezException("Container " + containerId
- + " has invalid request id. Expected: "
- + containerInfo.lastRequestId+1
- + " and actual: " + requestId);
- }
-
- List<TezEvent> inEvents = request.getEvents();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Ping from " + taskAttemptID.toString() +
- " events: " + (inEvents != null? inEvents.size() : -1));
- }
- if(inEvents!=null && !inEvents.isEmpty()) {
- TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
- context.getEventHandler().handle(
- new VertexEventRouteEvent(vertexId, inEvents));
- }
- taskHeartbeatHandler.pinged(taskAttemptID);
- List<TezEvent> outEvents = context
- .getCurrentDAG()
- .getVertex(taskAttemptID.getTaskID().getVertexID())
- .getTask(taskAttemptID.getTaskID())
- .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(),
- request.getMaxEvents());
- response.setEvents(outEvents);
- }
- containerInfo.lastRequestId = requestId;
- containerInfo.lastReponse = response;
- return response;
- }
- }
-
- private Map<String, TezLocalResource> convertLocalResourceMap(Map<String, LocalResource> ylrs)
- throws IOException {
- Map<String, TezLocalResource> tlrs = Maps.newHashMap();
- if (ylrs != null) {
- for (Entry<String, LocalResource> ylrEntry : ylrs.entrySet()) {
- TezLocalResource tlr;
- try {
- tlr = TezConverterUtils.convertYarnLocalResourceToTez(ylrEntry.getValue());
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- tlrs.put(ylrEntry.getKey(), tlr);
- }
- }
- return tlrs;
- }
-
- private ContainerTask getContainerTask(ContainerId containerId) throws IOException {
- ContainerTask containerTask = null;
- ContainerInfo containerInfo = registeredContainers.get(containerId);
- if (containerInfo == null) {
- // This can happen if an unregisterTask comes in after we've done the initial checks for
- // registered containers. (Race between getTask from the container, and a potential STOP_CONTAINER
- // from somewhere within the AM)
- // Implies that an un-registration has taken place and the container needs to be asked to die.
- LOG.info("Container with id: " + containerId
- + " is valid, but no longer registered, and will be killed");
- containerTask = TASK_FOR_INVALID_JVM;
- } else {
- synchronized (containerInfo) {
- if (containerInfo.amContainerTask != null) {
- if (!containerInfo.taskPulled) {
- containerInfo.taskPulled = true;
- AMContainerTask amContainerTask = containerInfo.amContainerTask;
- containerTask = new ContainerTask(amContainerTask.getTask(), false,
- convertLocalResourceMap(amContainerTask.getAdditionalResources()),
- amContainerTask.getCredentials(), amContainerTask.haveCredentialsChanged());
- } else {
- containerTask = null;
- }
- } else {
- containerTask = null;
- }
- }
- }
- return containerTask;
+ public TaskCommunicator getTaskCommunicator() {
+ return taskCommunicator;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
new file mode 100644
index 0000000..5652937
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java
@@ -0,0 +1,474 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.common.*;
+import org.apache.tez.common.ContainerContext;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.TaskCommunicator;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.api.TaskHeartbeatRequest;
+import org.apache.tez.dag.api.TaskHeartbeatResponse;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
+import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
+
+@InterfaceAudience.Private
+public class TezTaskCommunicatorImpl extends TaskCommunicator {
+
+ private static final Log LOG = LogFactory.getLog(TezTaskCommunicatorImpl.class);
+
+ private static final ContainerTask TASK_FOR_INVALID_JVM = new ContainerTask(
+ null, true, null, null, false);
+
+ private final TaskCommunicatorContext taskCommunicatorContext;
+
+ private final ConcurrentMap<ContainerId, ContainerInfo> registeredContainers =
+ new ConcurrentHashMap<ContainerId, ContainerInfo>();
+ private final ConcurrentMap<TaskAttempt, ContainerId> attemptToContainerMap =
+ new ConcurrentHashMap<TaskAttempt, ContainerId>();
+
+ private final TezTaskUmbilicalProtocol taskUmbilical;
+ private InetSocketAddress address;
+ private Server server;
+
+ private static final class ContainerInfo {
+
+ ContainerInfo(ContainerId containerId) {
+ this.containerId = containerId;
+ }
+
+ ContainerId containerId;
+ TezHeartbeatResponse lastResponse = null;
+ TaskSpec taskSpec = null;
+ long lastRequestId = 0;
+ Map<String, LocalResource> additionalLRs = null;
+ Credentials credentials = null;
+ boolean credentialsChanged = false;
+ boolean taskPulled = false;
+
+ void reset() {
+ taskSpec = null;
+ additionalLRs = null;
+ credentials = null;
+ credentialsChanged = false;
+ taskPulled = false;
+ }
+ }
+
+
+
+ /**
+ * Construct the service.
+ */
+ public TezTaskCommunicatorImpl(TaskCommunicatorContext taskCommunicatorContext) {
+ super(TezTaskCommunicatorImpl.class.getName());
+ this.taskCommunicatorContext = taskCommunicatorContext;
+ this.taskUmbilical = new TezTaskUmbilicalProtocolImpl();
+ }
+
+
+ @Override
+ public void serviceStart() {
+
+ startRpcServer();
+ }
+
+ @Override
+ public void serviceStop() {
+ stopRpcServer();
+ }
+
+ protected void startRpcServer() {
+ Configuration conf = getConfig();
+ if (!conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT)) {
+ try {
+ JobTokenSecretManager jobTokenSecretManager =
+ new JobTokenSecretManager();
+ Token<JobTokenIdentifier> sessionToken = TokenCache.getSessionToken(taskCommunicatorContext.getCredentials());
+ jobTokenSecretManager.addTokenForJob(
+ taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString(), sessionToken);
+
+ server = new RPC.Builder(conf)
+ .setProtocol(TezTaskUmbilicalProtocol.class)
+ .setBindAddress("0.0.0.0")
+ .setPort(0)
+ .setInstance(taskUmbilical)
+ .setNumHandlers(
+ conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
+ TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT))
+ .setSecretManager(jobTokenSecretManager).build();
+
+ // Enable service authorization?
+ if (conf.getBoolean(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
+ false)) {
+ refreshServiceAcls(conf, new TezAMPolicyProvider());
+ }
+
+ server.start();
+ this.address = NetUtils.getConnectAddress(server);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ } else {
+ try {
+ this.address = new InetSocketAddress(InetAddress.getLocalHost(), 0);
+ } catch (UnknownHostException e) {
+ throw new TezUncheckedException(e);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not starting TaskAttemptListener RPC in LocalMode");
+ }
+ }
+ }
+
+ protected void stopRpcServer() {
+ if (server != null) {
+ server.stop();
+ server = null;
+ }
+ }
+
+ private void refreshServiceAcls(Configuration configuration,
+ PolicyProvider policyProvider) {
+ this.server.refreshServiceAcl(configuration, policyProvider);
+ }
+
+ @Override
+ public void registerRunningContainer(ContainerId containerId, String host, int port) {
+ ContainerInfo oldInfo = registeredContainers.putIfAbsent(containerId, new ContainerInfo(containerId));
+ if (oldInfo != null) {
+ throw new TezUncheckedException("Multiple registrations for containerId: " + containerId);
+ }
+ }
+
+ @Override
+ public void registerContainerEnd(ContainerId containerId) {
+ ContainerInfo containerInfo = registeredContainers.remove(containerId);
+ if (containerInfo != null) {
+ synchronized(containerInfo) {
+ if (containerInfo.taskSpec != null && containerInfo.taskSpec.getTaskAttemptID() != null) {
+ attemptToContainerMap.remove(containerInfo.taskSpec.getTaskAttemptID());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void registerRunningTaskAttempt(ContainerId containerId, TaskSpec taskSpec,
+ Map<String, LocalResource> additionalResources,
+ Credentials credentials, boolean credentialsChanged) {
+
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ Preconditions.checkNotNull(containerInfo,
+ "Cannot register task attempt: " + taskSpec.getTaskAttemptID() + " to unknown container: " +
+ containerId);
+ synchronized (containerInfo) {
+ if (containerInfo.taskSpec != null) {
+ throw new TezUncheckedException(
+ "Cannot register task: " + taskSpec.getTaskAttemptID() + " to container: " +
+ containerId + " , with pre-existing assignment: " +
+ containerInfo.taskSpec.getTaskAttemptID());
+ }
+ containerInfo.taskSpec = taskSpec;
+ containerInfo.additionalLRs = additionalResources;
+ containerInfo.credentials = credentials;
+ containerInfo.credentialsChanged = credentialsChanged;
+ containerInfo.taskPulled = false;
+
+ ContainerId oldId = attemptToContainerMap.putIfAbsent(new TaskAttempt(taskSpec.getTaskAttemptID()), containerId);
+ if (oldId != null) {
+ throw new TezUncheckedException(
+ "Attempting to register an already registered taskAttempt with id: " +
+ taskSpec.getTaskAttemptID() + " to containerId: " + containerId +
+ ". Already registered to containerId: " + oldId);
+ }
+ }
+
+ }
+
+ @Override
+ public void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID) {
+ TaskAttempt taskAttempt = new TaskAttempt(taskAttemptID);
+ ContainerId containerId = attemptToContainerMap.remove(taskAttempt);
+ if(containerId == null) {
+ LOG.warn("Unregister task attempt: " + taskAttempt + " from unknown container");
+ return;
+ }
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ if (containerInfo == null) {
+ LOG.warn("Unregister task attempt: " + taskAttempt +
+ " from non-registered container: " + containerId);
+ return;
+ }
+ synchronized (containerInfo) {
+ containerInfo.reset();
+ attemptToContainerMap.remove(taskAttempt);
+ }
+ }
+
+ @Override
+ public InetSocketAddress getAddress() {
+ return address;
+ }
+
+ public TezTaskUmbilicalProtocol getUmbilical() {
+ return this.taskUmbilical;
+ }
+
+ private class TezTaskUmbilicalProtocolImpl implements TezTaskUmbilicalProtocol {
+
+ @Override
+ public ContainerTask getTask(ContainerContext containerContext) throws IOException {
+ ContainerTask task = null;
+ if (containerContext == null || containerContext.getContainerIdentifier() == null) {
+ LOG.info("Invalid task request with an empty containerContext or containerId");
+ task = TASK_FOR_INVALID_JVM;
+ } else {
+ ContainerId containerId = ConverterUtils.toContainerId(containerContext
+ .getContainerIdentifier());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Container with id: " + containerId + " asked for a task");
+ }
+ task = getContainerTask(containerId);
+ if (task != null && !task.shouldDie()) {
+ taskCommunicatorContext
+ .taskStartedRemotely(task.getTaskSpec().getTaskAttemptID(), containerId);
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getTask returning task: " + task);
+ }
+ return task;
+ }
+
+ @Override
+ public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
+ return taskCommunicatorContext.canCommit(taskAttemptId);
+ }
+
+ @Override
+ public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException,
+ TezException {
+ ContainerId containerId = ConverterUtils.toContainerId(request.getContainerIdentifier());
+ long requestId = request.getRequestId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received heartbeat from container"
+ + ", request=" + request);
+ }
+
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ if (containerInfo == null) {
+ LOG.warn("Received task heartbeat from unknown container with id: " + containerId +
+ ", asking it to die");
+ TezHeartbeatResponse response = new TezHeartbeatResponse();
+ response.setLastRequestId(requestId);
+ response.setShouldDie();
+ return response;
+ }
+
+ synchronized (containerInfo) {
+ if (containerInfo.lastRequestId == requestId) {
+ LOG.warn("Old sequenceId received: " + requestId
+ + ", Re-sending last response to client");
+ return containerInfo.lastResponse;
+ }
+ }
+
+ TaskHeartbeatResponse tResponse = null;
+
+
+ TezTaskAttemptID taskAttemptID = request.getCurrentTaskAttemptID();
+ if (taskAttemptID != null) {
+ synchronized (containerInfo) {
+ ContainerId containerIdFromMap = attemptToContainerMap.get(new TaskAttempt(taskAttemptID));
+ if (containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
+ throw new TezException("Attempt " + taskAttemptID
+ + " is not recognized for heartbeat");
+ }
+
+ if (containerInfo.lastRequestId + 1 != requestId) {
+ throw new TezException("Container " + containerId
+ + " has invalid request id. Expected: "
+ + containerInfo.lastRequestId + 1
+ + " and actual: " + requestId);
+ }
+ }
+ TaskHeartbeatRequest tRequest = new TaskHeartbeatRequest(request.getContainerIdentifier(),
+ request.getCurrentTaskAttemptID(), request.getEvents(), request.getStartIndex(),
+ request.getMaxEvents());
+ tResponse = taskCommunicatorContext.heartbeat(tRequest);
+ }
+ TezHeartbeatResponse response;
+ if (tResponse == null) {
+ response = new TezHeartbeatResponse();
+ } else {
+ response = new TezHeartbeatResponse(tResponse.getEvents());
+ }
+ response.setLastRequestId(requestId);
+ containerInfo.lastRequestId = requestId;
+ containerInfo.lastResponse = response;
+ return response;
+ }
+
+
+ // TODO Remove this method once we move to the Protobuf RPC engine
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+ return versionID;
+ }
+
+ // TODO Remove this method once we move to the Protobuf RPC engine
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
+ int clientMethodsHash) throws IOException {
+ return ProtocolSignature.getProtocolSignature(this, protocol,
+ clientVersion, clientMethodsHash);
+ }
+ }
+
+ private ContainerTask getContainerTask(ContainerId containerId) throws IOException {
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ ContainerTask task = null;
+ if (containerInfo == null) {
+ if (taskCommunicatorContext.isKnownContainer(containerId)) {
+ LOG.info("Container with id: " + containerId
+ + " is valid, but no longer registered, and will be killed");
+ } else {
+ LOG.info("Container with id: " + containerId
+ + " is invalid and will be killed");
+ }
+ task = TASK_FOR_INVALID_JVM;
+ } else {
+ synchronized (containerInfo) {
+ if (containerInfo.taskSpec != null) {
+ if (!containerInfo.taskPulled) {
+ containerInfo.taskPulled = true;
+ task = constructContainerTask(containerInfo);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Task " + containerInfo.taskSpec.getTaskAttemptID() +
+ " already sent to container: " + containerId);
+ }
+ task = null;
+ }
+ } else {
+ task = null;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No task assigned yet for running container: " + containerId);
+ }
+ }
+ }
+ }
+ return task;
+ }
+
+ private ContainerTask constructContainerTask(ContainerInfo containerInfo) throws IOException {
+ return new ContainerTask(containerInfo.taskSpec, false,
+ convertLocalResourceMap(containerInfo.additionalLRs), containerInfo.credentials,
+ containerInfo.credentialsChanged);
+ }
+
+ private Map<String, TezLocalResource> convertLocalResourceMap(Map<String, LocalResource> ylrs)
+ throws IOException {
+ Map<String, TezLocalResource> tlrs = Maps.newHashMap();
+ if (ylrs != null) {
+ for (Map.Entry<String, LocalResource> ylrEntry : ylrs.entrySet()) {
+ TezLocalResource tlr;
+ try {
+ tlr = TezConverterUtils.convertYarnLocalResourceToTez(ylrEntry.getValue());
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ tlrs.put(ylrEntry.getKey(), tlr);
+ }
+ }
+ return tlrs;
+ }
+
+
+ // Holder for Task information, which eventually will likely be VertexImplm taskIndex, attemptIndex
+ private static class TaskAttempt {
+ // TODO TEZ-2003 Change this to work with VertexName, int id, int version
+ // TODO TEZ-2003 Avoid constructing this unit all over the place
+ private TezTaskAttemptID taskAttemptId;
+
+ TaskAttempt(TezTaskAttemptID taskAttemptId) {
+ this.taskAttemptId = taskAttemptId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof TaskAttempt)) {
+ return false;
+ }
+
+ TaskAttempt that = (TaskAttempt) o;
+
+ if (!taskAttemptId.equals(that.taskAttemptId)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return taskAttemptId.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "TaskAttempt{" + "taskAttemptId=" + taskAttemptId + '}';
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 13601fe..7fc472b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -58,6 +58,8 @@ import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskAttemptListenerImpTezDag;
+import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
@@ -85,7 +87,7 @@ public class LocalContainerLauncher extends AbstractService implements
private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class);
private final AppContext context;
- private final TaskAttemptListener taskAttemptListener;
+ private final TezTaskUmbilicalProtocol taskUmbilicalProtocol;
private final AtomicBoolean serviceStopped = new AtomicBoolean(false);
private final String workingDirectory;
private final Map<String, String> localEnv = new HashMap<String, String>();
@@ -112,7 +114,9 @@ public class LocalContainerLauncher extends AbstractService implements
String workingDirectory) throws UnknownHostException {
super(LocalContainerLauncher.class.getName());
this.context = context;
- this.taskAttemptListener = taskAttemptListener;
+ TaskAttemptListenerImpTezDag taListener = (TaskAttemptListenerImpTezDag)taskAttemptListener;
+ TezTaskCommunicatorImpl taskComm = (TezTaskCommunicatorImpl) taListener.getTaskCommunicator();
+ this.taskUmbilicalProtocol = taskComm.getUmbilical();
this.workingDirectory = workingDirectory;
AuxiliaryServiceHelper.setServiceDataIntoEnv(
ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, ByteBuffer.allocate(4).putInt(0), localEnv);
@@ -205,7 +209,7 @@ public class LocalContainerLauncher extends AbstractService implements
tezChild =
createTezChild(context.getAMConf(), event.getContainerId(), tokenIdentifier,
context.getApplicationAttemptId().getAttemptId(), context.getLocalDirs(),
- (TezTaskUmbilicalProtocol) taskAttemptListener,
+ taskUmbilicalProtocol,
TezCommonUtils.parseCredentialsBytes(event.getContainerLaunchContext().getTokens().array()));
} catch (InterruptedException e) {
handleLaunchFailed(e, event.getContainerId());
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
index a6b403d..0fc2e12 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -32,5 +33,5 @@ public interface AMContainer extends EventHandler<AMContainerEvent>{
public Container getContainer();
public List<TezTaskAttemptID> getAllTaskAttempts();
public TezTaskAttemptID getCurrentTaskAttempt();
-
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
index a363168..9380cc9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
@@ -27,6 +27,8 @@ import org.apache.tez.runtime.api.impl.TaskSpec;
public class AMContainerEventAssignTA extends AMContainerEvent {
+ // TODO TEZ-2003. Add the task priority to this event.
+
private final TezTaskAttemptID attemptId;
// TODO Maybe have tht TAL pull the remoteTask from the TaskAttempt itself ?
private final TaskSpec remoteTaskSpec;
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index f72e62a..2aebaa4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index 04a47c6..cafcd04 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -27,6 +27,7 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.tez.client.TezApiVersionInfo;
import org.apache.tez.common.ContainerContext;
import org.apache.tez.common.ContainerTask;
+import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
import org.apache.tez.dag.app.launcher.ContainerLauncher;
@@ -86,6 +88,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
Map<ContainerId, ContainerData> containers = Maps.newConcurrentMap();
TaskAttemptListenerImpTezDag taListener;
+ TezTaskCommunicatorImpl taskCommunicator;
AtomicBoolean startScheduling = new AtomicBoolean(true);
AtomicBoolean goFlag;
@@ -126,6 +129,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
@Override
public void serviceStart() throws Exception {
taListener = (TaskAttemptListenerImpTezDag) getTaskAttemptListener();
+ taskCommunicator = (TezTaskCommunicatorImpl) taListener.getTaskCommunicator();
eventHandlingThread = new Thread(this);
eventHandlingThread.start();
}
@@ -236,7 +240,8 @@ public class MockDAGAppMaster extends DAGAppMaster {
if (cData.taId == null) {
// if container is not assigned a task, ask for a task
try {
- ContainerTask cTask = taListener.getTask(new ContainerContext(cId.toString()));
+ ContainerTask cTask =
+ taskCommunicator.getUmbilical().getTask(new ContainerContext(cId.toString()));
if (cTask == null) {
continue;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/a7a760fe/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
index f0f7dc5..843f88a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
@@ -1,16 +1,16 @@
/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.tez.dag.app;
@@ -18,6 +18,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
@@ -28,11 +29,14 @@ import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.common.ContainerContext;
import org.apache.tez.common.ContainerTask;
-import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.rm.container.AMContainer;
import org.apache.tez.dag.app.rm.container.AMContainerMap;
@@ -55,10 +59,18 @@ public class TestTaskAttemptListenerImplTezDag {
doReturn(dag).when(appContext).getCurrentDAG();
doReturn(appAcls).when(appContext).getApplicationACLs();
doReturn(amContainerMap).when(appContext).getAllContainers();
+ NodeId nodeId = NodeId.newInstance("localhost", 0);
+ AMContainer amContainer = mock(AMContainer.class);
+ Container container = mock(Container.class);
+ doReturn(nodeId).when(container).getNodeId();
+ doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
+ doReturn(container).when(amContainer).getContainer();
TaskAttemptListenerImpTezDag taskAttemptListener =
- new TaskAttemptListenerImplForTest(appContext, mock(TaskHeartbeatHandler.class),
+ new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
mock(ContainerHeartbeatHandler.class), null);
+ TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator();
+ TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
TaskSpec taskSpec = mock(TaskSpec.class);
@@ -69,33 +81,31 @@ public class TestTaskAttemptListenerImplTezDag {
ContainerId containerId1 = createContainerId(appId, 1);
- doReturn(mock(AMContainer.class)).when(amContainerMap).get(containerId1);
ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
- containerTask = taskAttemptListener.getTask(containerContext1);
+ containerTask = tezUmbilical.getTask(containerContext1);
assertTrue(containerTask.shouldDie());
ContainerId containerId2 = createContainerId(appId, 2);
- doReturn(mock(AMContainer.class)).when(amContainerMap).get(containerId2);
ContainerContext containerContext2 = new ContainerContext(containerId2.toString());
taskAttemptListener.registerRunningContainer(containerId2);
- containerTask = taskAttemptListener.getTask(containerContext2);
+ containerTask = tezUmbilical.getTask(containerContext2);
assertNull(containerTask);
// Valid task registered
taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2);
- containerTask = taskAttemptListener.getTask(containerContext2);
+ containerTask = tezUmbilical.getTask(containerContext2);
assertFalse(containerTask.shouldDie());
assertEquals(taskSpec, containerTask.getTaskSpec());
// Task unregistered. Should respond to heartbeats
taskAttemptListener.unregisterTaskAttempt(taskAttemptId);
- containerTask = taskAttemptListener.getTask(containerContext2);
+ containerTask = tezUmbilical.getTask(containerContext2);
assertNull(containerTask);
// Container unregistered. Should send a shouldDie = true
taskAttemptListener.unregisterRunningContainer(containerId2);
- containerTask = taskAttemptListener.getTask(containerContext2);
+ containerTask = tezUmbilical.getTask(containerContext2);
assertTrue(containerTask.shouldDie());
ContainerId containerId3 = createContainerId(appId, 3);
@@ -109,7 +119,7 @@ public class TestTaskAttemptListenerImplTezDag {
AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec, null, null, false);
taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3);
taskAttemptListener.unregisterRunningContainer(containerId3);
- containerTask = taskAttemptListener.getTask(containerContext3);
+ containerTask = tezUmbilical.getTask(containerContext3);
assertTrue(containerTask.shouldDie());
}
@@ -125,10 +135,18 @@ public class TestTaskAttemptListenerImplTezDag {
doReturn(dag).when(appContext).getCurrentDAG();
doReturn(appAcls).when(appContext).getApplicationACLs();
doReturn(amContainerMap).when(appContext).getAllContainers();
+ NodeId nodeId = NodeId.newInstance("localhost", 0);
+ AMContainer amContainer = mock(AMContainer.class);
+ Container container = mock(Container.class);
+ doReturn(nodeId).when(container).getNodeId();
+ doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
+ doReturn(container).when(amContainer).getContainer();
TaskAttemptListenerImpTezDag taskAttemptListener =
- new TaskAttemptListenerImplForTest(appContext, mock(TaskHeartbeatHandler.class),
+ new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
mock(ContainerHeartbeatHandler.class), null);
+ TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator();
+ TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
TaskSpec taskSpec = mock(TaskSpec.class);
@@ -139,20 +157,19 @@ public class TestTaskAttemptListenerImplTezDag {
ContainerId containerId1 = createContainerId(appId, 1);
- doReturn(mock(AMContainer.class)).when(amContainerMap).get(containerId1);
ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
taskAttemptListener.registerRunningContainer(containerId1);
- containerTask = taskAttemptListener.getTask(containerContext1);
+ containerTask = tezUmbilical.getTask(containerContext1);
assertNull(containerTask);
// Register task
taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1);
- containerTask = taskAttemptListener.getTask(containerContext1);
+ containerTask = tezUmbilical.getTask(containerContext1);
assertFalse(containerTask.shouldDie());
assertEquals(taskSpec, containerTask.getTaskSpec());
// Try pulling again - simulates re-use pull
- containerTask = taskAttemptListener.getTask(containerContext1);
+ containerTask = tezUmbilical.getTask(containerContext1);
assertNull(containerTask);
}
@@ -162,13 +179,11 @@ public class TestTaskAttemptListenerImplTezDag {
return containerId;
}
- private static class TaskAttemptListenerImplForTest extends TaskAttemptListenerImpTezDag {
+ private static class TezTaskCommunicatorImplForTest extends TezTaskCommunicatorImpl {
- public TaskAttemptListenerImplForTest(AppContext context,
- TaskHeartbeatHandler thh,
- ContainerHeartbeatHandler chh,
- JobTokenSecretManager jobTokenSecretManager) {
- super(context, thh, chh, jobTokenSecretManager);
+ public TezTaskCommunicatorImplForTest(
+ TaskCommunicatorContext taskCommunicatorContext) {
+ super(taskCommunicatorContext);
}
@Override
[22/23] tez git commit: TEZ-2090. Add tests for jobs running in
external services. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
new file mode 100644
index 0000000..4a6ce33
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
@@ -0,0 +1,512 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
+import org.apache.log4j.Logger;
+import org.apache.tez.common.TezCommonUtils;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.runtime.task.TaskReporter;
+import org.apache.tez.runtime.task.TezTaskRunner;
+import org.apache.tez.service.ContainerRunner;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.runtime.api.ExecutionContext;
+import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
+import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl;
+import org.apache.tez.runtime.task.TezChild;
+import org.apache.tez.runtime.task.TezChild.ContainerExecutionResult;
+import org.apache.tez.shufflehandler.ShuffleHandler;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto;
+import org.apache.tez.util.ProtoConverters;
+
+public class ContainerRunnerImpl extends AbstractService implements ContainerRunner {
+
+ private static final Logger LOG = Logger.getLogger(ContainerRunnerImpl.class);
+
+ private final ListeningExecutorService executorService;
+ private final AtomicReference<InetSocketAddress> localAddress;
+ private final String[] localDirsBase;
+ private final Map<String, String> localEnv = new HashMap<String, String>();
+ private volatile FileSystem localFs;
+ private final long memoryPerExecutor;
+ // TODO Support for removing queued containers, interrupting / killing specific containers - when preemption is supported
+
+
+
+
+ public ContainerRunnerImpl(int numExecutors, String[] localDirsBase,
+ AtomicReference<InetSocketAddress> localAddress,
+ long totalMemoryAvailableBytes) {
+ super("ContainerRunnerImpl");
+ Preconditions.checkState(numExecutors > 0,
+ "Invalid number of executors: " + numExecutors + ". Must be > 0");
+ this.localDirsBase = localDirsBase;
+ this.localAddress = localAddress;
+
+ ExecutorService raw = Executors.newFixedThreadPool(numExecutors,
+ new ThreadFactoryBuilder().setNameFormat("ContainerExecutor %d").build());
+ this.executorService = MoreExecutors.listeningDecorator(raw);
+
+
+ // 80% of memory considered for accounted buffers. Rest for objects.
+ // TODO Tune this based on the available size.
+ this.memoryPerExecutor = (long)(totalMemoryAvailableBytes * 0.8 / (float) numExecutors);
+
+ LOG.info("ContainerRunnerImpl config: " +
+ "memoryPerExecutorDerived=" + memoryPerExecutor +
+ ", numExecutors=" + numExecutors
+ );
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ try {
+ localFs = FileSystem.getLocal(conf);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to setup local filesystem instance", e);
+ }
+ }
+
+ @Override
+ public void serviceStart() {
+ }
+
+ public void setShufflePort(int shufflePort) {
+ AuxiliaryServiceHelper.setServiceDataIntoEnv(
+ TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
+ ByteBuffer.allocate(4).putInt(shufflePort), localEnv);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ }
+
+ // TODO Move this into a utilities class
+ private static String createAppSpecificLocalDir(String baseDir, String applicationIdString,
+ String user) {
+ return baseDir + File.separator + "usercache" + File.separator + user + File.separator +
+ "appcache" + File.separator + applicationIdString;
+ }
+
+ /**
+ * Submit a container which is ready for running.
+ * The regular pull mechanism will be used to fetch work from the AM
+ * @param request
+ * @throws IOException
+ */
+ @Override
+ public void queueContainer(RunContainerRequestProto request) throws IOException {
+ LOG.info("Queuing container for execution: " + request);
+
+ Map<String, String> env = new HashMap<String, String>();
+ env.putAll(localEnv);
+ env.put(ApplicationConstants.Environment.USER.name(), request.getUser());
+
+ String[] localDirs = new String[localDirsBase.length];
+
+ // Setup up local dirs to be application specific, and create them.
+ for (int i = 0; i < localDirsBase.length; i++) {
+ localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(),
+ request.getUser());
+ localFs.mkdirs(new Path(localDirs[i]));
+ }
+ LOG.info("DEBUG: Dirs are: " + Arrays.toString(localDirs));
+
+
+ // Setup workingDir. This is otherwise setup as Environment.PWD
+ // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream)
+ String workingDir = localDirs[0];
+
+ Credentials credentials = new Credentials();
+ DataInputBuffer dib = new DataInputBuffer();
+ byte[] tokenBytes = request.getCredentialsBinary().toByteArray();
+ dib.reset(tokenBytes, tokenBytes.length);
+ credentials.readTokenStorageStream(dib);
+
+ Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
+
+ // TODO Unregistering does not happen at the moment, since there's no signals on when an app completes.
+ LOG.info("DEBUG: Registering request with the ShuffleHandler");
+ ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser());
+
+
+ ContainerRunnerCallable callable = new ContainerRunnerCallable(request, new Configuration(getConfig()),
+ new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs,
+ workingDir, credentials, memoryPerExecutor);
+ ListenableFuture<ContainerExecutionResult> future = executorService
+ .submit(callable);
+ Futures.addCallback(future, new ContainerRunnerCallback(request, callable));
+ }
+
+ /**
+ * Submit an entire work unit - containerId + TaskSpec.
+ * This is intended for a task push from the AM
+ *
+ * @param request
+ * @throws IOException
+ */
+ @Override
+ public void submitWork(SubmitWorkRequestProto request) throws
+ IOException {
+ LOG.info("Queuing work for execution: " + request);
+
+ Map<String, String> env = new HashMap<String, String>();
+ env.putAll(localEnv);
+ env.put(ApplicationConstants.Environment.USER.name(), request.getUser());
+
+ String[] localDirs = new String[localDirsBase.length];
+
+ // Setup up local dirs to be application specific, and create them.
+ for (int i = 0; i < localDirsBase.length; i++) {
+ localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(),
+ request.getUser());
+ localFs.mkdirs(new Path(localDirs[i]));
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Dirs are: " + Arrays.toString(localDirs));
+ }
+
+ // Setup workingDir. This is otherwise setup as Environment.PWD
+ // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream)
+ String workingDir = localDirs[0];
+
+ Credentials credentials = new Credentials();
+ DataInputBuffer dib = new DataInputBuffer();
+ byte[] tokenBytes = request.getCredentialsBinary().toByteArray();
+ dib.reset(tokenBytes, tokenBytes.length);
+ credentials.readTokenStorageStream(dib);
+
+ Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
+
+ // TODO Unregistering does not happen at the moment, since there's no signals on when an app completes.
+ LOG.info("DEBUG: Registering request with the ShuffleHandler");
+ ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser());
+ TaskRunnerCallable callable = new TaskRunnerCallable(request, new Configuration(getConfig()),
+ new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs,
+ workingDir, credentials, memoryPerExecutor);
+ ListenableFuture<ContainerExecutionResult> future = executorService.submit(callable);
+ Futures.addCallback(future, new TaskRunnerCallback(request, callable));
+ }
+
+
+ static class ContainerRunnerCallable implements Callable<ContainerExecutionResult> {
+
+ private final RunContainerRequestProto request;
+ private final Configuration conf;
+ private final String workingDir;
+ private final String[] localDirs;
+ private final Map<String, String> envMap;
+ private final String pid = null;
+ private final ObjectRegistryImpl objectRegistry;
+ private final ExecutionContext executionContext;
+ private final Credentials credentials;
+ private final long memoryAvailable;
+ private volatile TezChild tezChild;
+
+
+ ContainerRunnerCallable(RunContainerRequestProto request, Configuration conf,
+ ExecutionContext executionContext, Map<String, String> envMap,
+ String[] localDirs, String workingDir, Credentials credentials,
+ long memoryAvailable) {
+ this.request = request;
+ this.conf = conf;
+ this.executionContext = executionContext;
+ this.envMap = envMap;
+ this.workingDir = workingDir;
+ this.localDirs = localDirs;
+ this.objectRegistry = new ObjectRegistryImpl();
+ this.credentials = credentials;
+ this.memoryAvailable = memoryAvailable;
+
+ }
+
+ @Override
+ public ContainerExecutionResult call() throws Exception {
+ Stopwatch sw = new Stopwatch().start();
+ tezChild =
+ new TezChild(conf, request.getAmHost(), request.getAmPort(),
+ request.getContainerIdString(),
+ request.getTokenIdentifier(), request.getAppAttemptNumber(), workingDir, localDirs,
+ envMap, objectRegistry, pid,
+ executionContext, credentials, memoryAvailable, request.getUser());
+ ContainerExecutionResult result = tezChild.run();
+ LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" +
+ sw.stop().elapsedMillis());
+ return result;
+ }
+
+ public TezChild getTezChild() {
+ return this.tezChild;
+ }
+ }
+
+
+ final class ContainerRunnerCallback implements FutureCallback<ContainerExecutionResult> {
+
+ private final RunContainerRequestProto request;
+ private final ContainerRunnerCallable containerRunnerCallable;
+
+ ContainerRunnerCallback(RunContainerRequestProto request,
+ ContainerRunnerCallable containerRunnerCallable) {
+ this.request = request;
+ this.containerRunnerCallable = containerRunnerCallable;
+ }
+
+ // TODO Proper error handling
+ @Override
+ public void onSuccess(ContainerExecutionResult result) {
+ switch (result.getExitStatus()) {
+ case SUCCESS:
+ LOG.info("Successfully finished: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString());
+ break;
+ case EXECUTION_FAILURE:
+ LOG.info("Failed to run: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), result.getThrowable());
+ break;
+ case INTERRUPTED:
+ LOG.info(
+ "Interrupted while running: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), result.getThrowable());
+ break;
+ case ASKED_TO_DIE:
+ LOG.info(
+ "Asked to die while running: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString());
+ break;
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.error(
+ "TezChild execution failed for : " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), t);
+ TezChild tezChild = containerRunnerCallable.getTezChild();
+ if (tezChild != null) {
+ tezChild.shutdown();
+ }
+ }
+ }
+
+ static class TaskRunnerCallable implements Callable<ContainerExecutionResult> {
+
+ private final SubmitWorkRequestProto request;
+ private final Configuration conf;
+ private final String workingDir;
+ private final String[] localDirs;
+ private final Map<String, String> envMap;
+ private final String pid = null;
+ private final ObjectRegistryImpl objectRegistry;
+ private final ExecutionContext executionContext;
+ private final Credentials credentials;
+ private final long memoryAvailable;
+ private final ListeningExecutorService executor;
+ private volatile TezTaskRunner taskRunner;
+ private volatile TaskReporter taskReporter;
+ private TezTaskUmbilicalProtocol umbilical;
+
+
+ TaskRunnerCallable(SubmitWorkRequestProto request, Configuration conf,
+ ExecutionContext executionContext, Map<String, String> envMap,
+ String[] localDirs, String workingDir, Credentials credentials,
+ long memoryAvailable) {
+ this.request = request;
+ this.conf = conf;
+ this.executionContext = executionContext;
+ this.envMap = envMap;
+ this.workingDir = workingDir;
+ this.localDirs = localDirs;
+ this.objectRegistry = new ObjectRegistryImpl();
+ this.credentials = credentials;
+ this.memoryAvailable = memoryAvailable;
+ // TODO This executor seems unnecessary. Here and TezChild
+ ExecutorService executorReal = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat("TezTaskRunner_" + request.getTaskSpec().getTaskAttemptIdString()).build());
+ executor = MoreExecutors.listeningDecorator(executorReal);
+ }
+
+ @Override
+ public ContainerExecutionResult call() throws Exception {
+
+ // TODO Consolidate this code with TezChild.
+ Stopwatch sw = new Stopwatch().start();
+ UserGroupInformation taskUgi = UserGroupInformation.createRemoteUser(request.getUser());
+ taskUgi.addCredentials(credentials);
+
+ Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
+ Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>();
+ serviceConsumerMetadata.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
+ TezCommonUtils.convertJobTokenToBytes(jobToken));
+ Multimap<String, String> startedInputsMap = HashMultimap.create();
+
+ UserGroupInformation taskOwner =
+ UserGroupInformation.createRemoteUser(request.getTokenIdentifier());
+ final InetSocketAddress address =
+ NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort());
+ SecurityUtil.setTokenService(jobToken, address);
+ taskOwner.addToken(jobToken);
+ umbilical = taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() {
+ @Override
+ public TezTaskUmbilicalProtocol run() throws Exception {
+ return RPC.getProxy(TezTaskUmbilicalProtocol.class,
+ TezTaskUmbilicalProtocol.versionID, address, conf);
+ }
+ });
+ // TODO Stop reading this on each request.
+ taskReporter = new TaskReporter(
+ umbilical,
+ conf.getInt(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS,
+ TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT),
+ conf.getLong(
+ TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS,
+ TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS_DEFAULT),
+ conf.getInt(TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT,
+ TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT),
+ new AtomicLong(0),
+ request.getContainerIdString());
+
+ taskRunner = new TezTaskRunner(conf, taskUgi, localDirs,
+ ProtoConverters.getTaskSpecfromProto(request.getTaskSpec()), umbilical,
+ request.getAppAttemptNumber(),
+ serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor, objectRegistry,
+ pid,
+ executionContext, memoryAvailable);
+
+ boolean shouldDie;
+ try {
+ shouldDie = !taskRunner.run();
+ if (shouldDie) {
+ LOG.info("Got a shouldDie notification via hearbeats. Shutting down");
+ return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null,
+ "Asked to die by the AM");
+ }
+ } catch (IOException e) {
+ return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
+ e, "TaskExecutionFailure: " + e.getMessage());
+ } catch (TezException e) {
+ return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
+ e, "TaskExecutionFailure: " + e.getMessage());
+ } finally {
+ FileSystem.closeAllForUGI(taskUgi);
+ }
+ LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" +
+ sw.stop().elapsedMillis());
+ return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null,
+ null);
+ }
+
+ public void shutdown() {
+ executor.shutdownNow();
+ if (taskReporter != null) {
+ taskReporter.shutdown();
+ }
+ if (umbilical != null) {
+ RPC.stopProxy(umbilical);
+ }
+ }
+ }
+
+
+ final class TaskRunnerCallback implements FutureCallback<ContainerExecutionResult> {
+
+ private final SubmitWorkRequestProto request;
+ private final TaskRunnerCallable taskRunnerCallable;
+
+ TaskRunnerCallback(SubmitWorkRequestProto request,
+ TaskRunnerCallable containerRunnerCallable) {
+ this.request = request;
+ this.taskRunnerCallable = containerRunnerCallable;
+ }
+
+ // TODO Proper error handling
+ @Override
+ public void onSuccess(ContainerExecutionResult result) {
+ switch (result.getExitStatus()) {
+ case SUCCESS:
+ LOG.info("Successfully finished: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString());
+ break;
+ case EXECUTION_FAILURE:
+ LOG.info("Failed to run: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), result.getThrowable());
+ break;
+ case INTERRUPTED:
+ LOG.info(
+ "Interrupted while running: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), result.getThrowable());
+ break;
+ case ASKED_TO_DIE:
+ LOG.info(
+ "Asked to die while running: " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString());
+ break;
+ }
+ taskRunnerCallable.shutdown();
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.error(
+ "TezTaskRunner execution failed for : " + request.getApplicationIdString() + ", containerId=" +
+ request.getContainerIdString(), t);
+ taskRunnerCallable.shutdown();
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
new file mode 100644
index 0000000..012e352
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.tez.service.ContainerRunner;
+import org.apache.tez.shufflehandler.ShuffleHandler;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+
+public class TezTestService extends AbstractService implements ContainerRunner {
+
+ private static final Logger LOG = Logger.getLogger(TezTestService.class);
+
+ private final Configuration shuffleHandlerConf;
+ private final int numExecutors;
+
+ private final TezTestServiceProtocolServerImpl server;
+ private final ContainerRunnerImpl containerRunner;
+ private final String[] localDirs;
+
+ private final AtomicInteger numSubmissions = new AtomicInteger(0);
+
+
+ private final AtomicReference<InetSocketAddress> address = new AtomicReference<InetSocketAddress>();
+
+ public TezTestService(Configuration conf, int numExecutors, long memoryAvailable, String[] localDirs) {
+ super(TezTestService.class.getSimpleName());
+ this.numExecutors = numExecutors;
+ this.localDirs = localDirs;
+
+ long memoryAvailableBytes = memoryAvailable;
+ long jvmMax = Runtime.getRuntime().maxMemory();
+
+ LOG.info(TezTestService.class.getSimpleName() + " created with the following configuration: " +
+ "numExecutors=" + numExecutors +
+ ", workDirs=" + Arrays.toString(localDirs) +
+ ", memoryAvailable=" + memoryAvailable +
+ ", jvmMaxMemory=" + jvmMax);
+
+ Preconditions.checkArgument(this.numExecutors > 0);
+ Preconditions.checkArgument(this.localDirs != null && this.localDirs.length > 0,
+ "Work dirs must be specified");
+ Preconditions.checkState(jvmMax >= memoryAvailableBytes,
+ "Invalid configuration. Xmx value too small. maxAvailable=" + jvmMax + ", configured=" +
+ memoryAvailableBytes);
+
+ this.shuffleHandlerConf = new Configuration(conf);
+ // Start Shuffle on a random port
+ this.shuffleHandlerConf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+ this.shuffleHandlerConf.set(ShuffleHandler.SHUFFLE_HANDLER_LOCAL_DIRS, StringUtils.arrayToString(localDirs));
+
+ this.server = new TezTestServiceProtocolServerImpl(this, address);
+ this.containerRunner = new ContainerRunnerImpl(numExecutors, localDirs, address,
+ memoryAvailableBytes);
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ server.init(conf);
+ containerRunner.init(conf);
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ ShuffleHandler.initializeAndStart(shuffleHandlerConf);
+ containerRunner.setShufflePort(ShuffleHandler.get().getPort());
+ server.start();
+ containerRunner.start();
+ }
+
+ public void serviceStop() throws Exception {
+ containerRunner.stop();
+ server.stop();
+ ShuffleHandler.get().stop();
+ }
+
+ public InetSocketAddress getListenerAddress() {
+ return server.getBindAddress();
+ }
+
+ public int getShufflePort() {
+ return ShuffleHandler.get().getPort();
+ }
+
+
+
+ @Override
+ public void queueContainer(RunContainerRequestProto request) throws IOException {
+ numSubmissions.incrementAndGet();
+ containerRunner.queueContainer(request);
+ }
+
+ @Override
+ public void submitWork(TezTestServiceProtocolProtos.SubmitWorkRequestProto request) throws
+ IOException {
+ numSubmissions.incrementAndGet();
+ containerRunner.submitWork(request);
+ }
+
+ public int getNumSubmissions() {
+ return numSubmissions.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java
new file mode 100644
index 0000000..10d2952
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.tez.service.TezTestServiceProtocolBlockingPB;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto;
+
+
+public class TezTestServiceProtocolClientImpl implements TezTestServiceProtocolBlockingPB {
+
+ private final Configuration conf;
+ private final InetSocketAddress serverAddr;
+ TezTestServiceProtocolBlockingPB proxy;
+
+
+ public TezTestServiceProtocolClientImpl(Configuration conf, String hostname, int port) {
+ this.conf = conf;
+ this.serverAddr = NetUtils.createSocketAddr(hostname, port);
+ }
+
+ @Override
+ public RunContainerResponseProto runContainer(RpcController controller,
+ RunContainerRequestProto request) throws
+ ServiceException {
+ try {
+ return getProxy().runContainer(null, request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public TezTestServiceProtocolProtos.SubmitWorkResponseProto submitWork(RpcController controller,
+ TezTestServiceProtocolProtos.SubmitWorkRequestProto request) throws
+ ServiceException {
+ try {
+ return getProxy().submitWork(null, request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+
+ public TezTestServiceProtocolBlockingPB getProxy() throws IOException {
+ if (proxy == null) {
+ proxy = createProxy();
+ }
+ return proxy;
+ }
+
+ public TezTestServiceProtocolBlockingPB createProxy() throws IOException {
+ TezTestServiceProtocolBlockingPB p;
+ // TODO Fix security
+ RPC.setProtocolEngine(conf, TezTestServiceProtocolBlockingPB.class, ProtobufRpcEngine.class);
+ p = (TezTestServiceProtocolBlockingPB) RPC
+ .getProxy(TezTestServiceProtocolBlockingPB.class, 0, serverAddr, conf);
+ return p;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
new file mode 100644
index 0000000..d7f8444
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.service.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.tez.service.ContainerRunner;
+import org.apache.tez.service.TezTestServiceProtocolBlockingPB;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto;
+
+public class TezTestServiceProtocolServerImpl extends AbstractService
+ implements TezTestServiceProtocolBlockingPB {
+
+ private static final Log LOG = LogFactory.getLog(TezTestServiceProtocolServerImpl.class);
+
+ private final ContainerRunner containerRunner;
+ private RPC.Server server;
+ private final AtomicReference<InetSocketAddress> bindAddress;
+
+
+ public TezTestServiceProtocolServerImpl(ContainerRunner containerRunner,
+ AtomicReference<InetSocketAddress> address) {
+ super(TezTestServiceProtocolServerImpl.class.getSimpleName());
+ this.containerRunner = containerRunner;
+ this.bindAddress = address;
+ }
+
+ @Override
+ public RunContainerResponseProto runContainer(RpcController controller,
+ RunContainerRequestProto request) throws
+ ServiceException {
+ LOG.info("Received request: " + request);
+ try {
+ containerRunner.queueContainer(request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ return RunContainerResponseProto.getDefaultInstance();
+ }
+
+ @Override
+ public SubmitWorkResponseProto submitWork(RpcController controller, TezTestServiceProtocolProtos.SubmitWorkRequestProto request) throws
+ ServiceException {
+ LOG.info("Received submitWork request: " + request);
+ try {
+ containerRunner.submitWork(request);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return SubmitWorkResponseProto.getDefaultInstance();
+ }
+
+
+ @Override
+ public void serviceStart() {
+ Configuration conf = getConfig();
+
+ int numHandlers = 3;
+ InetSocketAddress addr = new InetSocketAddress(0);
+
+ try {
+ server = createServer(TezTestServiceProtocolBlockingPB.class, addr, conf, numHandlers,
+ TezTestServiceProtocolProtos.TezTestServiceProtocol.newReflectiveBlockingService(this));
+ server.start();
+ } catch (IOException e) {
+ LOG.error("Failed to run RPC Server", e);
+ throw new RuntimeException(e);
+ }
+
+ InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server);
+ this.bindAddress.set(NetUtils.createSocketAddrForHost(
+ serverBindAddress.getAddress().getCanonicalHostName(),
+ serverBindAddress.getPort()));
+ LOG.info("Instantiated TestTestServiceListener at " + bindAddress);
+ }
+
+ @Override
+ public void serviceStop() {
+ if (server != null) {
+ server.stop();
+ }
+ }
+
+ @InterfaceAudience.Private
+ @VisibleForTesting
+ InetSocketAddress getBindAddress() {
+ return this.bindAddress.get();
+ }
+
+ private RPC.Server createServer(Class<?> pbProtocol, InetSocketAddress addr, Configuration conf,
+ int numHandlers, BlockingService blockingService) throws
+ IOException {
+ RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class);
+ RPC.Server server = new RPC.Builder(conf)
+ .setProtocol(pbProtocol)
+ .setInstance(blockingService)
+ .setBindAddress(addr.getHostName())
+ .setPort(0)
+ .setNumHandlers(numHandlers)
+ .build();
+ // TODO Add security.
+ return server;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java
new file mode 100644
index 0000000..65588fe
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedChunkedFile.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.shufflehandler;
+
+import java.io.FileDescriptor;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.ReadaheadPool;
+import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.jboss.netty.handler.stream.ChunkedFile;
+
+public class FadvisedChunkedFile extends ChunkedFile {
+
+ private static final Log LOG = LogFactory.getLog(FadvisedChunkedFile.class);
+
+ private final boolean manageOsCache;
+ private final int readaheadLength;
+ private final ReadaheadPool readaheadPool;
+ private final FileDescriptor fd;
+ private final String identifier;
+
+ private ReadaheadRequest readaheadRequest;
+
+ public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
+ int chunkSize, boolean manageOsCache, int readaheadLength,
+ ReadaheadPool readaheadPool, String identifier) throws IOException {
+ super(file, position, count, chunkSize);
+ this.manageOsCache = manageOsCache;
+ this.readaheadLength = readaheadLength;
+ this.readaheadPool = readaheadPool;
+ this.fd = file.getFD();
+ this.identifier = identifier;
+ }
+
+ @Override
+ public Object nextChunk() throws Exception {
+ if (manageOsCache && readaheadPool != null) {
+ readaheadRequest = readaheadPool
+ .readaheadStream(identifier, fd, getCurrentOffset(), readaheadLength,
+ getEndOffset(), readaheadRequest);
+ }
+ return super.nextChunk();
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (readaheadRequest != null) {
+ readaheadRequest.cancel();
+ }
+ if (manageOsCache && getEndOffset() - getStartOffset() > 0) {
+ try {
+ NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
+ fd,
+ getStartOffset(), getEndOffset() - getStartOffset(),
+ NativeIO.POSIX.POSIX_FADV_DONTNEED);
+ } catch (Throwable t) {
+ LOG.warn("Failed to manage OS cache for " + identifier, t);
+ }
+ }
+ super.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java
new file mode 100644
index 0000000..bdffe52
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/FadvisedFileRegion.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.shufflehandler;
+
+import java.io.FileDescriptor;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.ReadaheadPool;
+import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.jboss.netty.channel.DefaultFileRegion;
+
+public class FadvisedFileRegion extends DefaultFileRegion {
+
+ private static final Log LOG = LogFactory.getLog(FadvisedFileRegion.class);
+
+ private final boolean manageOsCache;
+ private final int readaheadLength;
+ private final ReadaheadPool readaheadPool;
+ private final FileDescriptor fd;
+ private final String identifier;
+ private final long count;
+ private final long position;
+ private final int shuffleBufferSize;
+ private final boolean shuffleTransferToAllowed;
+ private final FileChannel fileChannel;
+
+ private ReadaheadRequest readaheadRequest;
+
+ public FadvisedFileRegion(RandomAccessFile file, long position, long count,
+ boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
+ String identifier, int shuffleBufferSize,
+ boolean shuffleTransferToAllowed) throws IOException {
+ super(file.getChannel(), position, count);
+ this.manageOsCache = manageOsCache;
+ this.readaheadLength = readaheadLength;
+ this.readaheadPool = readaheadPool;
+ this.fd = file.getFD();
+ this.identifier = identifier;
+ this.fileChannel = file.getChannel();
+ this.count = count;
+ this.position = position;
+ this.shuffleBufferSize = shuffleBufferSize;
+ this.shuffleTransferToAllowed = shuffleTransferToAllowed;
+ }
+
+ @Override
+ public long transferTo(WritableByteChannel target, long position)
+ throws IOException {
+ if (manageOsCache && readaheadPool != null) {
+ readaheadRequest = readaheadPool.readaheadStream(identifier, fd,
+ getPosition() + position, readaheadLength,
+ getPosition() + getCount(), readaheadRequest);
+ }
+
+ if(this.shuffleTransferToAllowed) {
+ return super.transferTo(target, position);
+ } else {
+ return customShuffleTransfer(target, position);
+ }
+ }
+
+ /**
+ * This method transfers data using local buffer. It transfers data from
+ * a disk to a local buffer in memory, and then it transfers data from the
+ * buffer to the target. This is used only if transferTo is disallowed in
+ * the configuration file. super.TransferTo does not perform well on Windows
+ * due to a small IO request generated. customShuffleTransfer can control
+ * the size of the IO requests by changing the size of the intermediate
+ * buffer.
+ */
+ @VisibleForTesting
+ long customShuffleTransfer(WritableByteChannel target, long position)
+ throws IOException {
+ long actualCount = this.count - position;
+ if (actualCount < 0 || position < 0) {
+ throw new IllegalArgumentException(
+ "position out of range: " + position +
+ " (expected: 0 - " + (this.count - 1) + ')');
+ }
+ if (actualCount == 0) {
+ return 0L;
+ }
+
+ long trans = actualCount;
+ int readSize;
+ ByteBuffer byteBuffer = ByteBuffer.allocate(this.shuffleBufferSize);
+
+ while(trans > 0L &&
+ (readSize = fileChannel.read(byteBuffer, this.position+position)) > 0) {
+ //adjust counters and buffer limit
+ if(readSize < trans) {
+ trans -= readSize;
+ position += readSize;
+ byteBuffer.flip();
+ } else {
+ //We can read more than we need if the actualCount is not multiple
+ //of the byteBuffer size and file is big enough. In that case we cannot
+ //use flip method but we need to set buffer limit manually to trans.
+ byteBuffer.limit((int)trans);
+ byteBuffer.position(0);
+ position += trans;
+ trans = 0;
+ }
+
+ //write data to the target
+ while(byteBuffer.hasRemaining()) {
+ target.write(byteBuffer);
+ }
+
+ byteBuffer.clear();
+ }
+
+ return actualCount - trans;
+ }
+
+
+ @Override
+ public void releaseExternalResources() {
+ if (readaheadRequest != null) {
+ readaheadRequest.cancel();
+ }
+ super.releaseExternalResources();
+ }
+
+ /**
+ * Call when the transfer completes successfully so we can advise the OS that
+ * we don't need the region to be cached anymore.
+ */
+ public void transferSuccessful() {
+ if (manageOsCache && getCount() > 0) {
+ try {
+ NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
+ fd, getPosition(), getCount(),
+ NativeIO.POSIX.POSIX_FADV_DONTNEED);
+ } catch (Throwable t) {
+ LOG.warn("Failed to manage OS cache for " + identifier, t);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java
new file mode 100644
index 0000000..9a51ca0
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.shufflehandler;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.tez.runtime.library.common.Constants;
+import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
+import org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord;
+
+class IndexCache {
+
+ private final Configuration conf;
+ private final int totalMemoryAllowed;
+ private AtomicInteger totalMemoryUsed = new AtomicInteger();
+ private static final Log LOG = LogFactory.getLog(IndexCache.class);
+
+ private final ConcurrentHashMap<String,IndexInformation> cache =
+ new ConcurrentHashMap<String,IndexInformation>();
+
+ private final LinkedBlockingQueue<String> queue =
+ new LinkedBlockingQueue<String>();
+
+ public IndexCache(Configuration conf) {
+ this.conf = conf;
+ totalMemoryAllowed = 10 * 1024 * 1024;
+ LOG.info("IndexCache created with max memory = " + totalMemoryAllowed);
+ }
+
+ /**
+ * This method gets the index information for the given mapId and reduce.
+ * It reads the index file into cache if it is not already present.
+ * @param mapId
+ * @param reduce
+ * @param fileName The file to read the index information from if it is not
+ * already present in the cache
+ * @param expectedIndexOwner The expected owner of the index file
+ * @return The Index Information
+ * @throws IOException
+ */
+ public TezIndexRecord getIndexInformation(String mapId, int reduce,
+ Path fileName, String expectedIndexOwner)
+ throws IOException {
+
+ IndexInformation info = cache.get(mapId);
+
+ if (info == null) {
+ info = readIndexFileToCache(fileName, mapId, expectedIndexOwner);
+ } else {
+ synchronized(info) {
+ while (isUnderConstruction(info)) {
+ try {
+ info.wait();
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for construction", e);
+ }
+ }
+ }
+ LOG.debug("IndexCache HIT: MapId " + mapId + " found");
+ }
+
+ if (info.mapSpillRecord.size() == 0 ||
+ info.mapSpillRecord.size() <= reduce) {
+ throw new IOException("Invalid request " +
+ " Map Id = " + mapId + " Reducer = " + reduce +
+ " Index Info Length = " + info.mapSpillRecord.size());
+ }
+ return info.mapSpillRecord.getIndex(reduce);
+ }
+
+ private boolean isUnderConstruction(IndexInformation info) {
+ synchronized(info) {
+ return (null == info.mapSpillRecord);
+ }
+ }
+
+ private IndexInformation readIndexFileToCache(Path indexFileName,
+ String mapId,
+ String expectedIndexOwner)
+ throws IOException {
+ IndexInformation info;
+ IndexInformation newInd = new IndexInformation();
+ if ((info = cache.putIfAbsent(mapId, newInd)) != null) {
+ synchronized(info) {
+ while (isUnderConstruction(info)) {
+ try {
+ info.wait();
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for construction", e);
+ }
+ }
+ }
+ LOG.debug("IndexCache HIT: MapId " + mapId + " found");
+ return info;
+ }
+ LOG.debug("IndexCache MISS: MapId " + mapId + " not found") ;
+ TezSpillRecord tmp = null;
+ try {
+ tmp = new TezSpillRecord(indexFileName, conf, expectedIndexOwner);
+ } catch (Throwable e) {
+ tmp = new TezSpillRecord(0);
+ cache.remove(mapId);
+ throw new IOException("Error Reading IndexFile", e);
+ } finally {
+ synchronized (newInd) {
+ newInd.mapSpillRecord = tmp;
+ newInd.notifyAll();
+ }
+ }
+ queue.add(mapId);
+
+ if (totalMemoryUsed.addAndGet(newInd.getSize()) > totalMemoryAllowed) {
+ freeIndexInformation();
+ }
+ return newInd;
+ }
+
+ /**
+ * This method removes the map from the cache if index information for this
+ * map is loaded(size>0), index information entry in cache will not be
+ * removed if it is in the loading phrase(size=0), this prevents corruption
+ * of totalMemoryUsed. It should be called when a map output on this tracker
+ * is discarded.
+ * @param mapId The taskID of this map.
+ */
+ public void removeMap(String mapId) {
+ IndexInformation info = cache.get(mapId);
+ if (info == null || ((info != null) && isUnderConstruction(info))) {
+ return;
+ }
+ info = cache.remove(mapId);
+ if (info != null) {
+ totalMemoryUsed.addAndGet(-info.getSize());
+ if (!queue.remove(mapId)) {
+ LOG.warn("Map ID" + mapId + " not found in queue!!");
+ }
+ } else {
+ LOG.info("Map ID " + mapId + " not found in cache");
+ }
+ }
+
+ /**
+ * This method checks if cache and totolMemoryUsed is consistent.
+ * It is only used for unit test.
+ * @return True if cache and totolMemoryUsed is consistent
+ */
+ boolean checkTotalMemoryUsed() {
+ int totalSize = 0;
+ for (IndexInformation info : cache.values()) {
+ totalSize += info.getSize();
+ }
+ return totalSize == totalMemoryUsed.get();
+ }
+
+ /**
+ * Bring memory usage below totalMemoryAllowed.
+ */
+ private synchronized void freeIndexInformation() {
+ while (totalMemoryUsed.get() > totalMemoryAllowed) {
+ String s = queue.remove();
+ IndexInformation info = cache.remove(s);
+ if (info != null) {
+ totalMemoryUsed.addAndGet(-info.getSize());
+ }
+ }
+ }
+
+ private static class IndexInformation {
+ TezSpillRecord mapSpillRecord;
+
+ int getSize() {
+ return mapSpillRecord == null
+ ? 0
+ : mapSpillRecord.size() * Constants.MAP_OUTPUT_INDEX_RECORD_LENGTH;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java
new file mode 100644
index 0000000..cc82d74
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java
@@ -0,0 +1,840 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.shufflehandler;
+
+import static org.jboss.netty.buffer.ChannelBuffers.wrappedBuffer;
+import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
+import static org.jboss.netty.handler.codec.http.HttpMethod.GET;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.METHOD_NOT_ALLOWED;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
+import static org.jboss.netty.handler.codec.http.HttpResponseStatus.UNAUTHORIZED;
+import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+
+import javax.crypto.SecretKey;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.ReadaheadPool;
+import org.apache.hadoop.io.SecureIOUtils;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.MutableCounterInt;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.runtime.library.common.security.SecureShuffleUtils;
+import org.apache.tez.runtime.library.common.shuffle.orderedgrouped.ShuffleHeader;
+import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
+import org.jboss.netty.bootstrap.ServerBootstrap;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelPipelineFactory;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
+import org.jboss.netty.channel.group.ChannelGroup;
+import org.jboss.netty.channel.group.DefaultChannelGroup;
+import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
+import org.jboss.netty.handler.codec.frame.TooLongFrameException;
+import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
+import org.jboss.netty.handler.codec.http.HttpChunkAggregator;
+import org.jboss.netty.handler.codec.http.HttpHeaders;
+import org.jboss.netty.handler.codec.http.HttpRequest;
+import org.jboss.netty.handler.codec.http.HttpRequestDecoder;
+import org.jboss.netty.handler.codec.http.HttpResponse;
+import org.jboss.netty.handler.codec.http.HttpResponseEncoder;
+import org.jboss.netty.handler.codec.http.HttpResponseStatus;
+import org.jboss.netty.handler.codec.http.QueryStringDecoder;
+import org.jboss.netty.handler.ssl.SslHandler;
+import org.jboss.netty.handler.stream.ChunkedWriteHandler;
+import org.jboss.netty.util.CharsetUtil;
+
+public class ShuffleHandler {
+
+ private static final Log LOG = LogFactory.getLog(ShuffleHandler.class);
+
+ public static final String SHUFFLE_HANDLER_LOCAL_DIRS = "tez.shuffle.handler.local-dirs";
+
+ public static final String SHUFFLE_MANAGE_OS_CACHE = "mapreduce.shuffle.manage.os.cache";
+ public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true;
+
+ public static final String SHUFFLE_READAHEAD_BYTES = "mapreduce.shuffle.readahead.bytes";
+ public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES = 4 * 1024 * 1024;
+
+ // pattern to identify errors related to the client closing the socket early
+ // idea borrowed from Netty SslHandler
+ private static final Pattern IGNORABLE_ERROR_MESSAGE = Pattern.compile(
+ "^.*(?:connection.*reset|connection.*closed|broken.*pipe).*$",
+ Pattern.CASE_INSENSITIVE);
+
+ private int port;
+ private final ChannelFactory selector;
+ private final ChannelGroup accepted = new DefaultChannelGroup();
+ protected HttpPipelineFactory pipelineFact;
+ private final int sslFileBufferSize;
+ private final Configuration conf;
+
+ private final ConcurrentMap<String, Boolean> registeredApps = new ConcurrentHashMap<String, Boolean>();
+
+ /**
+ * Should the shuffle use posix_fadvise calls to manage the OS cache during
+ * sendfile
+ */
+ private final boolean manageOsCache;
+ private final int readaheadLength;
+ private final int maxShuffleConnections;
+ private final int shuffleBufferSize;
+ private final boolean shuffleTransferToAllowed;
+ private final ReadaheadPool readaheadPool = ReadaheadPool.getInstance();
+
+ private Map<String,String> userRsrc;
+ private JobTokenSecretManager secretManager;
+
+ // TODO Fix this for tez.
+ public static final String MAPREDUCE_SHUFFLE_SERVICEID =
+ "mapreduce_shuffle";
+
+ public static final String SHUFFLE_PORT_CONFIG_KEY = "tez.shuffle.port";
+ public static final int DEFAULT_SHUFFLE_PORT = 15551;
+
+ // TODO Change configs to remove mapreduce references.
+ public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED =
+ "mapreduce.shuffle.connection-keep-alive.enable";
+ public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false;
+
+ public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT =
+ "mapreduce.shuffle.connection-keep-alive.timeout";
+ public static final int DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT = 5; //seconds
+
+ public static final String SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
+ "mapreduce.shuffle.mapoutput-info.meta.cache.size";
+ public static final int DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
+ 1000;
+
+ public static final String CONNECTION_CLOSE = "close";
+
+ public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY =
+ "mapreduce.shuffle.ssl.file.buffer.size";
+
+ public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024;
+
+ public static final String MAX_SHUFFLE_CONNECTIONS = "mapreduce.shuffle.max.connections";
+ public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS = 0; // 0 implies no limit
+
+ public static final String MAX_SHUFFLE_THREADS = "mapreduce.shuffle.max.threads";
+ // 0 implies Netty default of 2 * number of available processors
+ public static final int DEFAULT_MAX_SHUFFLE_THREADS = 0;
+
+ public static final String SHUFFLE_BUFFER_SIZE =
+ "mapreduce.shuffle.transfer.buffer.size";
+ public static final int DEFAULT_SHUFFLE_BUFFER_SIZE = 128 * 1024;
+
+ public static final String SHUFFLE_TRANSFERTO_ALLOWED =
+ "mapreduce.shuffle.transferTo.allowed";
+ public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true;
+ public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED =
+ false;
+
+ final boolean connectionKeepAliveEnabled;
+ final int connectionKeepAliveTimeOut;
+ final int mapOutputMetaInfoCacheSize;
+ private static final AtomicBoolean started = new AtomicBoolean(false);
+ private static final AtomicBoolean initing = new AtomicBoolean(false);
+ private static ShuffleHandler INSTANCE;
+
+ @Metrics(about="Shuffle output metrics", context="mapred")
+ static class ShuffleMetrics implements ChannelFutureListener {
+ @Metric("Shuffle output in bytes")
+ MutableCounterLong shuffleOutputBytes;
+ @Metric("# of failed shuffle outputs")
+ MutableCounterInt shuffleOutputsFailed;
+ @Metric("# of succeeeded shuffle outputs")
+ MutableCounterInt shuffleOutputsOK;
+ @Metric("# of current shuffle connections")
+ MutableGaugeInt shuffleConnections;
+
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ if (future.isSuccess()) {
+ shuffleOutputsOK.incr();
+ } else {
+ shuffleOutputsFailed.incr();
+ }
+ shuffleConnections.decr();
+ }
+ }
+
+ public ShuffleHandler(Configuration conf) {
+ this.conf = conf;
+ manageOsCache = conf.getBoolean(SHUFFLE_MANAGE_OS_CACHE,
+ DEFAULT_SHUFFLE_MANAGE_OS_CACHE);
+
+ readaheadLength = conf.getInt(SHUFFLE_READAHEAD_BYTES,
+ DEFAULT_SHUFFLE_READAHEAD_BYTES);
+
+ maxShuffleConnections = conf.getInt(MAX_SHUFFLE_CONNECTIONS,
+ DEFAULT_MAX_SHUFFLE_CONNECTIONS);
+ int maxShuffleThreads = conf.getInt(MAX_SHUFFLE_THREADS,
+ DEFAULT_MAX_SHUFFLE_THREADS);
+ if (maxShuffleThreads == 0) {
+ maxShuffleThreads = 2 * Runtime.getRuntime().availableProcessors();
+ }
+
+ shuffleBufferSize = conf.getInt(SHUFFLE_BUFFER_SIZE,
+ DEFAULT_SHUFFLE_BUFFER_SIZE);
+
+ shuffleTransferToAllowed = conf.getBoolean(SHUFFLE_TRANSFERTO_ALLOWED,
+ (Shell.WINDOWS)?WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED:
+ DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED);
+
+ ThreadFactory bossFactory = new ThreadFactoryBuilder()
+ .setNameFormat("ShuffleHandler Netty Boss #%d")
+ .build();
+ ThreadFactory workerFactory = new ThreadFactoryBuilder()
+ .setNameFormat("ShuffleHandler Netty Worker #%d")
+ .build();
+
+ selector = new NioServerSocketChannelFactory(
+ Executors.newCachedThreadPool(bossFactory),
+ Executors.newCachedThreadPool(workerFactory),
+ maxShuffleThreads);
+
+ sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY,
+ DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE);
+ connectionKeepAliveEnabled =
+ conf.getBoolean(SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED,
+ DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED);
+ connectionKeepAliveTimeOut =
+ Math.max(1, conf.getInt(SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
+ DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT));
+ mapOutputMetaInfoCacheSize =
+ Math.max(1, conf.getInt(SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE,
+ DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE));
+
+ userRsrc = new ConcurrentHashMap<String,String>();
+ secretManager = new JobTokenSecretManager();
+ }
+
+
+ public void start() throws Exception {
+ ServerBootstrap bootstrap = new ServerBootstrap(selector);
+ try {
+ pipelineFact = new HttpPipelineFactory(conf);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ bootstrap.setPipelineFactory(pipelineFact);
+ port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
+ Channel ch = bootstrap.bind(new InetSocketAddress(port));
+ accepted.add(ch);
+ port = ((InetSocketAddress)ch.getLocalAddress()).getPort();
+ conf.set(SHUFFLE_PORT_CONFIG_KEY, Integer.toString(port));
+ pipelineFact.SHUFFLE.setPort(port);
+ LOG.info("TezShuffleHandler" + " listening on port " + port);
+ }
+
+ public static void initializeAndStart(Configuration conf) throws Exception {
+ if (!initing.getAndSet(true)) {
+ INSTANCE = new ShuffleHandler(conf);
+ INSTANCE.start();
+ started.set(true);
+ }
+ }
+
+ public static ShuffleHandler get() {
+ Preconditions.checkState(started.get(), "ShuffleHandler must be started before invoking started");
+ return INSTANCE;
+ }
+
+ /**
+ * Serialize the shuffle port into a ByteBuffer for use later on.
+ * @param port the port to be sent to the ApplciationMaster
+ * @return the serialized form of the port.
+ */
+ public static ByteBuffer serializeMetaData(int port) throws IOException {
+ //TODO these bytes should be versioned
+ DataOutputBuffer port_dob = new DataOutputBuffer();
+ port_dob.writeInt(port);
+ return ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength());
+ }
+
+ /**
+ * A helper function to deserialize the metadata returned by ShuffleHandler.
+ * @param meta the metadata returned by the ShuffleHandler
+ * @return the port the Shuffle Handler is listening on to serve shuffle data.
+ */
+ public static int deserializeMetaData(ByteBuffer meta) throws IOException {
+ //TODO this should be returning a class not just an int
+ DataInputByteBuffer in = new DataInputByteBuffer();
+ in.reset(meta);
+ int port = in.readInt();
+ return port;
+ }
+
+ /**
+ * A helper function to serialize the JobTokenIdentifier to be sent to the
+ * ShuffleHandler as ServiceData.
+ * @param jobToken the job token to be used for authentication of
+ * shuffle data requests.
+ * @return the serialized version of the jobToken.
+ */
+ public static ByteBuffer serializeServiceData(Token<JobTokenIdentifier> jobToken) throws IOException {
+ //TODO these bytes should be versioned
+ DataOutputBuffer jobToken_dob = new DataOutputBuffer();
+ jobToken.write(jobToken_dob);
+ return ByteBuffer.wrap(jobToken_dob.getData(), 0, jobToken_dob.getLength());
+ }
+
+ static Token<JobTokenIdentifier> deserializeServiceData(ByteBuffer secret) throws IOException {
+ DataInputByteBuffer in = new DataInputByteBuffer();
+ in.reset(secret);
+ Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>();
+ jt.readFields(in);
+ return jt;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public void registerApplication(String applicationIdString, Token<JobTokenIdentifier> appToken,
+ String user) {
+ Boolean registered = registeredApps.putIfAbsent(applicationIdString, Boolean.valueOf(true));
+ if (registered == null) {
+ recordJobShuffleInfo(applicationIdString, user, appToken);
+ }
+ }
+
+ public void unregisterApplication(String applicationIdString) {
+ removeJobShuffleInfo(applicationIdString);
+ }
+
+
+ public void stop() throws Exception {
+ accepted.close().awaitUninterruptibly(10, TimeUnit.SECONDS);
+ if (selector != null) {
+ ServerBootstrap bootstrap = new ServerBootstrap(selector);
+ bootstrap.releaseExternalResources();
+ }
+ if (pipelineFact != null) {
+ pipelineFact.destroy();
+ }
+ }
+
+ protected Shuffle getShuffle(Configuration conf) {
+ return new Shuffle(conf);
+ }
+
+
+ private void addJobToken(String appIdString, String user,
+ Token<JobTokenIdentifier> jobToken) {
+ String jobIdString = appIdString.replace("application", "job");
+ userRsrc.put(jobIdString, user);
+ secretManager.addTokenForJob(jobIdString, jobToken);
+ LOG.info("Added token for " + jobIdString);
+ }
+
+ private void recordJobShuffleInfo(String appIdString, String user,
+ Token<JobTokenIdentifier> jobToken) {
+ addJobToken(appIdString, user, jobToken);
+ }
+
+ private void removeJobShuffleInfo(String appIdString) {
+ secretManager.removeTokenForJob(appIdString);
+ userRsrc.remove(appIdString);
+ }
+
+ class HttpPipelineFactory implements ChannelPipelineFactory {
+
+ final Shuffle SHUFFLE;
+ private SSLFactory sslFactory;
+
+ public HttpPipelineFactory(Configuration conf) throws Exception {
+ SHUFFLE = getShuffle(conf);
+ // TODO Setup SSL Shuffle
+// if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY,
+// MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) {
+// LOG.info("Encrypted shuffle is enabled.");
+// sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf);
+// sslFactory.init();
+// }
+ }
+
+ public void destroy() {
+ if (sslFactory != null) {
+ sslFactory.destroy();
+ }
+ }
+
+ @Override
+ public ChannelPipeline getPipeline() throws Exception {
+ ChannelPipeline pipeline = Channels.pipeline();
+ if (sslFactory != null) {
+ pipeline.addLast("ssl", new SslHandler(sslFactory.createSSLEngine()));
+ }
+ pipeline.addLast("decoder", new HttpRequestDecoder());
+ pipeline.addLast("aggregator", new HttpChunkAggregator(1 << 16));
+ pipeline.addLast("encoder", new HttpResponseEncoder());
+ pipeline.addLast("chunking", new ChunkedWriteHandler());
+ pipeline.addLast("shuffle", SHUFFLE);
+ return pipeline;
+ // TODO factor security manager into pipeline
+ // TODO factor out encode/decode to permit binary shuffle
+ // TODO factor out decode of index to permit alt. models
+ }
+
+ }
+
+ class Shuffle extends SimpleChannelUpstreamHandler {
+
+ private final Configuration conf;
+ private final IndexCache indexCache;
+ private final LocalDirAllocator lDirAlloc =
+ new LocalDirAllocator(SHUFFLE_HANDLER_LOCAL_DIRS);
+ private int port;
+
+ public Shuffle(Configuration conf) {
+ this.conf = conf;
+ indexCache = new IndexCache(conf);
+ this.port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
+ }
+
+ public void setPort(int port) {
+ this.port = port;
+ }
+
+ private List<String> splitMaps(List<String> mapq) {
+ if (null == mapq) {
+ return null;
+ }
+ final List<String> ret = new ArrayList<String>();
+ for (String s : mapq) {
+ Collections.addAll(ret, s.split(","));
+ }
+ return ret;
+ }
+
+ @Override
+ public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent evt)
+ throws Exception {
+ if ((maxShuffleConnections > 0) && (accepted.size() >= maxShuffleConnections)) {
+ LOG.info(String.format("Current number of shuffle connections (%d) is " +
+ "greater than or equal to the max allowed shuffle connections (%d)",
+ accepted.size(), maxShuffleConnections));
+ evt.getChannel().close();
+ return;
+ }
+ accepted.add(evt.getChannel());
+ super.channelOpen(ctx, evt);
+
+ }
+
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt)
+ throws Exception {
+ HttpRequest request = (HttpRequest) evt.getMessage();
+ if (request.getMethod() != GET) {
+ sendError(ctx, METHOD_NOT_ALLOWED);
+ return;
+ }
+ // Check whether the shuffle version is compatible
+ if (!ShuffleHeader.DEFAULT_HTTP_HEADER_NAME.equals(
+ request.getHeader(ShuffleHeader.HTTP_HEADER_NAME))
+ || !ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION.equals(
+ request.getHeader(ShuffleHeader.HTTP_HEADER_VERSION))) {
+ sendError(ctx, "Incompatible shuffle request version", BAD_REQUEST);
+ }
+ final Map<String,List<String>> q =
+ new QueryStringDecoder(request.getUri()).getParameters();
+ final List<String> keepAliveList = q.get("keepAlive");
+ boolean keepAliveParam = false;
+ if (keepAliveList != null && keepAliveList.size() == 1) {
+ keepAliveParam = Boolean.valueOf(keepAliveList.get(0));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("KeepAliveParam : " + keepAliveList
+ + " : " + keepAliveParam);
+ }
+ }
+ final List<String> mapIds = splitMaps(q.get("map"));
+ final List<String> reduceQ = q.get("reduce");
+ final List<String> jobQ = q.get("job");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("RECV: " + request.getUri() +
+ "\n mapId: " + mapIds +
+ "\n reduceId: " + reduceQ +
+ "\n jobId: " + jobQ +
+ "\n keepAlive: " + keepAliveParam);
+ }
+
+ if (mapIds == null || reduceQ == null || jobQ == null) {
+ sendError(ctx, "Required param job, map and reduce", BAD_REQUEST);
+ return;
+ }
+ if (reduceQ.size() != 1 || jobQ.size() != 1) {
+ sendError(ctx, "Too many job/reduce parameters", BAD_REQUEST);
+ return;
+ }
+ int reduceId;
+ String jobId;
+ try {
+ reduceId = Integer.parseInt(reduceQ.get(0));
+ jobId = jobQ.get(0);
+ } catch (NumberFormatException e) {
+ sendError(ctx, "Bad reduce parameter", BAD_REQUEST);
+ return;
+ } catch (IllegalArgumentException e) {
+ sendError(ctx, "Bad job parameter", BAD_REQUEST);
+ return;
+ }
+ final String reqUri = request.getUri();
+ if (null == reqUri) {
+ // TODO? add upstream?
+ sendError(ctx, FORBIDDEN);
+ return;
+ }
+ HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
+ try {
+ verifyRequest(jobId, ctx, request, response,
+ new URL("http", "", this.port, reqUri));
+ } catch (IOException e) {
+ LOG.warn("Shuffle failure ", e);
+ sendError(ctx, e.getMessage(), UNAUTHORIZED);
+ return;
+ }
+
+ Map<String, MapOutputInfo> mapOutputInfoMap =
+ new HashMap<String, MapOutputInfo>();
+ Channel ch = evt.getChannel();
+ String user = userRsrc.get(jobId);
+
+ // $x/$user/appcache/$appId/output/$mapId
+ // TODO: Once Shuffle is out of NM, this can use MR APIs to convert
+ // between App and Job
+ String outputBasePathStr = getBaseLocation(jobId, user);
+
+ try {
+ populateHeaders(mapIds, outputBasePathStr, user, reduceId, request,
+ response, keepAliveParam, mapOutputInfoMap);
+ } catch(IOException e) {
+ ch.write(response);
+ LOG.error("Shuffle error in populating headers :", e);
+ String errorMessage = getErrorMessage(e);
+ sendError(ctx,errorMessage , INTERNAL_SERVER_ERROR);
+ return;
+ }
+ ch.write(response);
+ // TODO refactor the following into the pipeline
+ ChannelFuture lastMap = null;
+ for (String mapId : mapIds) {
+ try {
+ MapOutputInfo info = mapOutputInfoMap.get(mapId);
+ if (info == null) {
+ info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+ }
+ lastMap =
+ sendMapOutput(ctx, ch, user, mapId,
+ reduceId, info);
+ if (null == lastMap) {
+ sendError(ctx, NOT_FOUND);
+ return;
+ }
+ } catch (IOException e) {
+ LOG.error("Shuffle error :", e);
+ String errorMessage = getErrorMessage(e);
+ sendError(ctx,errorMessage , INTERNAL_SERVER_ERROR);
+ return;
+ }
+ }
+ lastMap.addListener(ChannelFutureListener.CLOSE);
+ }
+
+ private String getErrorMessage(Throwable t) {
+ StringBuffer sb = new StringBuffer(t.getMessage());
+ while (t.getCause() != null) {
+ sb.append(t.getCause().getMessage());
+ t = t.getCause();
+ }
+ return sb.toString();
+ }
+
+ private final String USERCACHE_CONSTANT = "usercache";
+ private final String APPCACHE_CONSTANT = "appcache";
+
+ private String getBaseLocation(String jobIdString, String user) {
+ String parts[] = jobIdString.split("_");
+ Preconditions.checkArgument(parts.length == 3, "Invalid jobId. Expecting 3 parts");
+ final ApplicationId appID =
+ ApplicationId.newInstance(Long.parseLong(parts[1]), Integer.parseInt(parts[2]));
+ final String baseStr =
+ USERCACHE_CONSTANT + "/" + user + "/"
+ + APPCACHE_CONSTANT + "/"
+ + ConverterUtils.toString(appID) + "/output" + "/";
+ return baseStr;
+ }
+
+ protected MapOutputInfo getMapOutputInfo(String base, String mapId,
+ int reduce, String user) throws IOException {
+ // Index file
+ Path indexFileName =
+ lDirAlloc.getLocalPathToRead(base + "/file.out.index", conf);
+ TezIndexRecord info =
+ indexCache.getIndexInformation(mapId, reduce, indexFileName, user);
+
+ Path mapOutputFileName =
+ lDirAlloc.getLocalPathToRead(base + "/file.out", conf);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(base + " : " + mapOutputFileName + " : " + indexFileName);
+ }
+ MapOutputInfo outputInfo = new MapOutputInfo(mapOutputFileName, info);
+ return outputInfo;
+ }
+
+ protected void populateHeaders(List<String> mapIds, String outputBaseStr,
+ String user, int reduce, HttpRequest request, HttpResponse response,
+ boolean keepAliveParam, Map<String, MapOutputInfo> mapOutputInfoMap)
+ throws IOException {
+
+ long contentLength = 0;
+ for (String mapId : mapIds) {
+ String base = outputBaseStr + mapId;
+ MapOutputInfo outputInfo = getMapOutputInfo(base, mapId, reduce, user);
+ if (mapOutputInfoMap.size() < mapOutputMetaInfoCacheSize) {
+ mapOutputInfoMap.put(mapId, outputInfo);
+ }
+ // Index file
+ Path indexFileName =
+ lDirAlloc.getLocalPathToRead(base + "/file.out.index", conf);
+ TezIndexRecord info =
+ indexCache.getIndexInformation(mapId, reduce, indexFileName, user);
+ ShuffleHeader header =
+ new ShuffleHeader(mapId, info.getPartLength(), info.getRawLength(), reduce);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ header.write(dob);
+
+ contentLength += info.getPartLength();
+ contentLength += dob.getLength();
+ }
+
+ // Now set the response headers.
+ setResponseHeaders(response, keepAliveParam, contentLength);
+ }
+
+ protected void setResponseHeaders(HttpResponse response,
+ boolean keepAliveParam, long contentLength) {
+ if (!connectionKeepAliveEnabled && !keepAliveParam) {
+ LOG.info("Setting connection close header...");
+ response.setHeader(HttpHeaders.Names.CONNECTION, CONNECTION_CLOSE);
+ } else {
+ response.setHeader(HttpHeaders.Names.CONTENT_LENGTH,
+ String.valueOf(contentLength));
+ response.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
+ response.setHeader(HttpHeaders.Values.KEEP_ALIVE, "timeout="
+ + connectionKeepAliveTimeOut);
+ LOG.info("Content Length in shuffle : " + contentLength);
+ }
+ }
+
+ class MapOutputInfo {
+ final Path mapOutputFileName;
+ final TezIndexRecord indexRecord;
+
+ MapOutputInfo(Path mapOutputFileName, TezIndexRecord indexRecord) {
+ this.mapOutputFileName = mapOutputFileName;
+ this.indexRecord = indexRecord;
+ }
+ }
+
+ protected void verifyRequest(String appid, ChannelHandlerContext ctx,
+ HttpRequest request, HttpResponse response, URL requestUri)
+ throws IOException {
+ SecretKey tokenSecret = secretManager.retrieveTokenSecret(appid);
+ if (null == tokenSecret) {
+ LOG.info("Request for unknown token " + appid);
+ throw new IOException("could not find jobid");
+ }
+ // string to encrypt
+ String enc_str = SecureShuffleUtils.buildMsgFrom(requestUri);
+ // hash from the fetcher
+ String urlHashStr =
+ request.getHeader(SecureShuffleUtils.HTTP_HEADER_URL_HASH);
+ if (urlHashStr == null) {
+ LOG.info("Missing header hash for " + appid);
+ throw new IOException("fetcher cannot be authenticated");
+ }
+ if (LOG.isDebugEnabled()) {
+ int len = urlHashStr.length();
+ LOG.debug("verifying request. enc_str=" + enc_str + "; hash=..." +
+ urlHashStr.substring(len-len/2, len-1));
+ }
+ // verify - throws exception
+ SecureShuffleUtils.verifyReply(urlHashStr, enc_str, tokenSecret);
+ // verification passed - encode the reply
+ String reply =
+ SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8),
+ tokenSecret);
+ response.setHeader(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH, reply);
+ // Put shuffle version into http header
+ response.setHeader(ShuffleHeader.HTTP_HEADER_NAME,
+ ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
+ response.setHeader(ShuffleHeader.HTTP_HEADER_VERSION,
+ ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
+ if (LOG.isDebugEnabled()) {
+ int len = reply.length();
+ LOG.debug("Fetcher request verfied. enc_str=" + enc_str + ";reply=" +
+ reply.substring(len-len/2, len-1));
+ }
+ }
+
+ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch,
+ String user, String mapId, int reduce, MapOutputInfo mapOutputInfo)
+ throws IOException {
+ final TezIndexRecord info = mapOutputInfo.indexRecord;
+ final ShuffleHeader header =
+ new ShuffleHeader(mapId, info.getPartLength(), info.getRawLength(), reduce);
+ final DataOutputBuffer dob = new DataOutputBuffer();
+ header.write(dob);
+ ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+ final File spillfile =
+ new File(mapOutputInfo.mapOutputFileName.toString());
+ RandomAccessFile spill;
+ try {
+ spill = SecureIOUtils.openForRandomRead(spillfile, "r", user, null);
+ } catch (FileNotFoundException e) {
+ LOG.info(spillfile + " not found");
+ return null;
+ }
+ ChannelFuture writeFuture;
+ if (ch.getPipeline().get(SslHandler.class) == null) {
+ final FadvisedFileRegion partition = new FadvisedFileRegion(spill,
+ info.getStartOffset(), info.getPartLength(), manageOsCache, readaheadLength,
+ readaheadPool, spillfile.getAbsolutePath(),
+ shuffleBufferSize, shuffleTransferToAllowed);
+ writeFuture = ch.write(partition);
+ writeFuture.addListener(new ChannelFutureListener() {
+ // TODO error handling; distinguish IO/connection failures,
+ // attribute to appropriate spill output
+ @Override
+ public void operationComplete(ChannelFuture future) {
+ if (future.isSuccess()) {
+ partition.transferSuccessful();
+ }
+ partition.releaseExternalResources();
+ }
+ });
+ } else {
+ // HTTPS cannot be done with zero copy.
+ final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill,
+ info.getStartOffset(), info.getPartLength(), sslFileBufferSize,
+ manageOsCache, readaheadLength, readaheadPool,
+ spillfile.getAbsolutePath());
+ writeFuture = ch.write(chunk);
+ }
+ return writeFuture;
+ }
+
+ protected void sendError(ChannelHandlerContext ctx,
+ HttpResponseStatus status) {
+ sendError(ctx, "", status);
+ }
+
+ protected void sendError(ChannelHandlerContext ctx, String message,
+ HttpResponseStatus status) {
+ HttpResponse response = new DefaultHttpResponse(HTTP_1_1, status);
+ response.setHeader(CONTENT_TYPE, "text/plain; charset=UTF-8");
+ // Put shuffle version into http header
+ response.setHeader(ShuffleHeader.HTTP_HEADER_NAME,
+ ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
+ response.setHeader(ShuffleHeader.HTTP_HEADER_VERSION,
+ ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
+ response.setContent(
+ ChannelBuffers.copiedBuffer(message, CharsetUtil.UTF_8));
+
+ // Close the connection as soon as the error message is sent.
+ ctx.getChannel().write(response).addListener(ChannelFutureListener.CLOSE);
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e)
+ throws Exception {
+ Channel ch = e.getChannel();
+ Throwable cause = e.getCause();
+ if (cause instanceof TooLongFrameException) {
+ sendError(ctx, BAD_REQUEST);
+ return;
+ } else if (cause instanceof IOException) {
+ if (cause instanceof ClosedChannelException) {
+ LOG.debug("Ignoring closed channel error", cause);
+ return;
+ }
+ String message = String.valueOf(cause.getMessage());
+ if (IGNORABLE_ERROR_MESSAGE.matcher(message).matches()) {
+ LOG.debug("Ignoring client socket close", cause);
+ return;
+ }
+ }
+
+ LOG.error("Shuffle error: ", cause);
+ if (ch.isConnected()) {
+ LOG.error("Shuffle error " + e);
+ sendError(ctx, INTERNAL_SERVER_ERROR);
+ }
+ }
+ }
+}
[18/23] tez git commit: TEZ-2080. Localclient should be using tezconf
in init instead of yarnconf. (sseth)
Posted by ss...@apache.org.
TEZ-2080. Localclient should be using tezconf in init instead of
yarnconf. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/269905b1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/269905b1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/269905b1
Branch: refs/heads/TEZ-2003
Commit: 269905b112f35b5014d60bacd9d8962c16e2fc8d
Parents: 88d2325
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Feb 18 11:25:41 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Feb 18 11:25:41 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
tez-dag/src/main/java/org/apache/tez/client/LocalClient.java | 6 +++---
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/269905b1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 27509d1..ff16163 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.7.0: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2080. Localclient should be using tezconf in init instead of yarnconf.
TEZ-2072. Add missing Private annotation to createDAG in the DAG API class.
TEZ-2095. master branch fails to compile against hadoop-2.4.
TEZ-2093. Add events to MockDAGAppMaster and add e2e test for event routing
http://git-wip-us.apache.org/repos/asf/tez/blob/269905b1/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
index 8649a4d..36301fb 100644
--- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
+++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
@@ -76,11 +76,11 @@ public class LocalClient extends FrameworkClient {
@Override
public void init(TezConfiguration tezConf, YarnConfiguration yarnConf) {
- this.conf = yarnConf;
+ this.conf = tezConf;
tezConf.set("fs.defaultFS", "file:///");
// Tez libs already in the client's classpath
- tezConf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
- tezConf.set(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, localModeDAGSchedulerClassName);
+ this.conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
+ this.conf.set(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, localModeDAGSchedulerClassName);
isSession = tezConf.getBoolean(TezConfiguration.TEZ_AM_SESSION_MODE,
TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT);
[13/23] tez git commit: TEZ-2114. Tez UI: task/task attempt status is
not available when its running. (Prakash Ramachandran via hitesh)
Posted by ss...@apache.org.
TEZ-2114. Tez UI: task/task attempt status is not available when its running. (Prakash Ramachandran via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ec7ad206
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ec7ad206
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ec7ad206
Branch: refs/heads/TEZ-2003
Commit: ec7ad2069677a8cdf07500a174ca940c659b100e
Parents: 1653d18
Author: Hitesh Shah <hi...@apache.org>
Authored: Tue Feb 17 12:38:32 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Tue Feb 17 12:38:32 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../ats/HistoryEventTimelineConversion.java | 4 +++
.../ats/TestHistoryEventTimelineConversion.java | 4 +++
.../scripts/components/dag-view/graph-view.js | 2 +-
.../controllers/dag-task-attempts-controller.js | 12 +++++--
.../scripts/controllers/dag-view-controller.js | 7 +++-
.../app/scripts/controllers/dag_controller.js | 4 +--
.../scripts/controllers/dag_index_controller.js | 2 +-
.../webapp/app/scripts/controllers/dag_tasks.js | 19 ++++++++--
.../app/scripts/controllers/dag_vertices.js | 8 +++--
.../app/scripts/controllers/dags_controller.js | 5 +--
.../controllers/task_index_controller.js | 12 ++++---
.../task_task_attempts_controller.js | 15 +++++---
.../app/scripts/controllers/tasks_controller.js | 5 +--
.../controllers/tez-app-dags-controller.js | 5 +--
.../scripts/controllers/vertex_controller.js | 22 ++++++------
.../controllers/vertex_index_controller.js | 2 +-
.../vertex_task_attempts_controller.js | 14 +++++---
.../controllers/vertex_tasks_controller.js | 17 +++++++--
.../src/main/webapp/app/scripts/helpers/misc.js | 37 +++++++++++++++++---
tez-ui/src/main/webapp/app/scripts/router.js | 3 ++
.../main/webapp/app/templates/task/index.hbs | 2 +-
.../webapp/app/templates/taskAttempt/index.hbs | 4 +--
23 files changed, 151 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5458feb..cc481c2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -64,6 +64,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2114. Tez UI: task/task attempt status is not available when its running.
TEZ-2112. Tez UI: fix offset calculation, add home button to breadcrumbs.
TEZ-2038. TEZ-UI DAG is always running in tez-ui when the app is failed but no DAGFinishedEvent is logged.
TEZ-2102. Tez UI: DAG view has hidden edges, dragging DAG by holding vertex causes unintended click.
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
index 9d85ee8..ca47b92 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.tez.common.ATSConstants;
import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.history.events.AMLaunchedEvent;
@@ -435,6 +437,7 @@ public class HistoryEventTimelineConversion {
atsEntity.addOtherInfo(ATSConstants.NODE_ID, event.getNodeId().toString());
atsEntity.addOtherInfo(ATSConstants.NODE_HTTP_ADDRESS, event.getNodeHttpAddress());
atsEntity.addOtherInfo(ATSConstants.CONTAINER_ID, event.getContainerId().toString());
+ atsEntity.addOtherInfo(ATSConstants.STATUS, TaskAttemptState.RUNNING.name());
return atsEntity;
}
@@ -497,6 +500,7 @@ public class HistoryEventTimelineConversion {
atsEntity.addOtherInfo(ATSConstants.START_TIME, event.getStartTime());
atsEntity.addOtherInfo(ATSConstants.SCHEDULED_TIME, event.getScheduledTime());
+ atsEntity.addOtherInfo(ATSConstants.STATUS, TaskState.SCHEDULED.name());
return atsEntity;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index d4b9e69..6d713c5 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -705,6 +705,8 @@ public class TestHistoryEventTimelineConversion {
((Long)timelineEntity.getOtherInfo().get(ATSConstants.SCHEDULED_TIME)).longValue());
Assert.assertEquals(startTime,
((Long)timelineEntity.getOtherInfo().get(ATSConstants.START_TIME)).longValue());
+ Assert.assertTrue(TaskState.SCHEDULED.name()
+ .equals(timelineEntity.getOtherInfo().get(ATSConstants.STATUS)));
}
@Test(timeout = 5000)
@@ -759,6 +761,8 @@ public class TestHistoryEventTimelineConversion {
timelineEntity.getOtherInfo().get(ATSConstants.CONTAINER_ID));
Assert.assertEquals("nodeHttpAddress",
timelineEntity.getOtherInfo().get(ATSConstants.NODE_HTTP_ADDRESS));
+ Assert.assertTrue(TaskAttemptState.RUNNING.name()
+ .equals(timelineEntity.getOtherInfo().get(ATSConstants.STATUS)));
}
@Test(timeout = 5000)
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js b/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
index 88c11f7..1bfb13a 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/dag-view/graph-view.js
@@ -221,7 +221,7 @@ App.DagViewComponent.graphView = (function (){
*/
function _addStatusBar(node, d) {
var group = node.append('g'),
- statusIcon = App.Helpers.misc.getStatusClassForEntity(d.get('data'));
+ statusIcon = App.Helpers.misc.getStatusClassForEntity(d.get('data.status'));
group.attr('class', 'status-bar');
group.append('foreignObject')
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/dag-task-attempts-controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag-task-attempts-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag-task-attempts-controller.js
index d151af8..50df0bf 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag-task-attempts-controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag-task-attempts-controller.js
@@ -54,6 +54,7 @@ App.DagTaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentM
defaultColumnConfigs: function() {
var that = this;
+ var dagStatus = this.get('controllers.dag.status');
return [
{
id: 'taskId',
@@ -117,7 +118,7 @@ App.DagTaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentM
headerCellName: 'Status',
filterID: 'status_filter',
filterType: 'dropdown',
- dropdownValues: App.Helpers.misc.taskStatusUIOptions,
+ dropdownValues: App.Helpers.misc.taskAttemptStatusUIOptions,
tableCellViewClass: Em.Table.TableCell.extend({
template: Em.Handlebars.compile(
'<span class="ember-table-content"> \
@@ -125,9 +126,14 @@ App.DagTaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentM
{{view.cellContent.status}}</span>')
}),
getCellContent: function(row) {
+ var status = App.Helpers.misc.getFixedupDisplayStatus(row.get('status'));
+ if (status == 'RUNNING' &&
+ App.Helpers.misc.isStatusInUnsuccessful(dagStatus)) {
+ status = 'KILLED';
+ }
return {
- status: row.get('status'),
- statusIcon: App.Helpers.misc.getStatusClassForEntity(row)
+ status: status,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(status)
};
}
},
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/dag-view-controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag-view-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag-view-controller.js
index 9e73718..31753d1 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag-view-controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag-view-controller.js
@@ -74,7 +74,9 @@ App.DagViewController = Em.ObjectController.extend(App.PaginatedContentMixin, Ap
viewData: function () {
var vertices = this.get('controllers.dag.vertices'),
entities = this.get('entities'),
- finalVertex;
+ finalVertex,
+ dagStatus = this.get('controllers.dag.status'),
+ needsStatusFixup = App.Helpers.misc.isStatusInUnsuccessful(dagStatus);
entities = entities.reduce(function (obj, vertexData) {
obj[vertexData.get('name')] = vertexData;
@@ -83,6 +85,9 @@ App.DagViewController = Em.ObjectController.extend(App.PaginatedContentMixin, Ap
vertices.forEach(function (vertex) {
vertex.data = entities[vertex.vertexName];
+ if (needsStatusFixup && vertex.data.get('status') == 'RUNNING') {
+ vertex.data.set('status', 'KILLED');
+ }
});
return {
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
index 210e673..c8d9c87 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
@@ -45,11 +45,11 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
dag.set('yarnAppState', appState);
}
dag.set('status', App.Helpers.misc.getRealStatus(dag.get('status'), app.get('appState'), app.get('finalAppStatus')));
- });
+ }).catch(function(){});
var tezAppLoader = this.store.find('tezApp', 'tez_' + applicationId)
.then(function(app){
dag.set('tezApp', app);
- });
+ }).catch(function(){});
loaders.push(appDetailLoader);
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
index fed28c7..3261787 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_index_controller.js
@@ -21,7 +21,7 @@ App.DagIndexController = Em.ObjectController.extend({
controllerName: 'DagIndexController',
taskIconStatus: function() {
- return App.Helpers.misc.getStatusClassForEntity(this.get('model'));
+ return App.Helpers.misc.getStatusClassForEntity(this.get('model.status'));
}.property('id', 'status', 'counterGroups'),
progressStr: function() {
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
index 849b1db..93b9234 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
@@ -57,9 +57,21 @@ App.DagTasksController = Em.ObjectController.extend(App.PaginatedContentMixin, A
.fmt(App.env.timelineBaseUrl);
store.unloadAll(childEntityType);
- store.findQuery(childEntityType, this.getFilterProperties()).then(function(entities){
+ store.findQuery(childEntityType, this.getFilterProperties())
+ .then(function(entities){
+
var pivotLoaders = [];
+ var dagStatus = that.get('controllers.dag.status');
entities.forEach(function (task) {
+ var taskStatus = App.Helpers.misc
+ .getFixedupDisplayStatus(task.get('status'));
+ if (taskStatus == 'RUNNING' &&
+ App.Helpers.misc.isStatusInUnsuccessful(dagStatus)) {
+ taskStatus = 'KILLED'
+ }
+ if (taskStatus != task.get('status')) {
+ task.set('status', taskStatus);
+ }
var taskAttemptId = task.get('successfulAttemptId') ||
task.get('attempts.lastObject');
if (!!taskAttemptId) {
@@ -158,9 +170,10 @@ App.DagTasksController = Em.ObjectController.extend(App.PaginatedContentMixin, A
{{view.cellContent.status}}</span>')
}),
getCellContent: function(row) {
+ var status = row.get('status');
return {
- status: row.get('status'),
- statusIcon: App.Helpers.misc.getStatusClassForEntity(row)
+ status: status,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(status)
};
}
},
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
index a373220..f63524a 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
@@ -176,13 +176,15 @@ App.DagVerticesController = Em.ObjectController.extend(App.PaginatedContentMixin
{{#if view.cellContent.progress}} {{bs-badge content=view.cellContent.progress}}{{/if}}</span>')
}),
getCellContent: function(row) {
- var pct;
+ var pct,
+ vertexStatus = row.get('status');
if (Ember.typeOf(row.get('progress')) === 'number') {
pct = App.Helpers.number.fractionToPercentage(row.get('progress'));
}
+
return {
- status: row.get('status'),
- statusIcon: App.Helpers.misc.getStatusClassForEntity(row),
+ status: vertexStatus,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(vertexStatus),
progress: pct
};
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
index 53d9eeb..35a0ecc 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
@@ -179,9 +179,10 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
if (Ember.typeOf(row.get('progress')) === 'number') {
pct = App.Helpers.number.fractionToPercentage(row.get('progress'));
}
+ var dagStatus = row.get('status');
return {
- status: row.get('status'),
- statusIcon: App.Helpers.misc.getStatusClassForEntity(row),
+ status: dagStatus,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(dagStatus),
progress: pct
};
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/task_index_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/task_index_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/task_index_controller.js
index fa36314..7f3eeb3 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/task_index_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/task_index_controller.js
@@ -18,10 +18,14 @@
//TODO: watch individual counters.
App.TaskIndexController = Em.ObjectController.extend({
- controllerName: 'TaskIndexController',
+ controllerName: 'TaskIndexController',
- taskIconStatus: function() {
- return App.Helpers.misc.getStatusClassForEntity(this.get('model'));
- }.property('id', 'status', 'counterGroups'),
+ taskStatus: function() {
+ return App.Helpers.misc.getFixedupDisplayStatus(this.get('model.status'));
+ }.property('id', 'status'),
+
+ taskIconStatus: function() {
+ return App.Helpers.misc.getStatusClassForEntity(this.get('taskStatus'));
+ }.property('id', 'status', 'counterGroups'),
});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js
index 84de0a7..0749e00 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/task_task_attempts_controller.js
@@ -161,7 +161,7 @@ App.TaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentMixi
headerCellName: 'Status',
filterID: 'status_filter',
filterType: 'dropdown',
- dropdownValues: App.Helpers.misc.taskStatusUIOptions,
+ dropdownValues: App.Helpers.misc.taskAttemptStatusUIOptions,
tableCellViewClass: Em.Table.TableCell.extend({
template: Em.Handlebars.compile(
'<span class="ember-table-content"> \
@@ -169,9 +169,10 @@ App.TaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentMixi
{{view.cellContent.status}}</span>')
}),
getCellContent: function(row) {
+ var status = App.Helpers.misc.getFixedupDisplayStatus(row.get('status'));
return {
- status: row.get('status'),
- statusIcon: App.Helpers.misc.getStatusClassForEntity(row)
+ status: status,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(status)
};
}
},
@@ -255,8 +256,12 @@ App.TaskAttemptsController = Em.ObjectController.extend(App.PaginatedContentMixi
App.TaskAttemptIndexController = Em.ObjectController.extend({
controllerName: 'TaskAttemptIndexController',
- taskIconStatus: function() {
- return App.Helpers.misc.getStatusClassForEntity(this.get('model'));
+ taskAttemptStatus: function() {
+ return App.Helpers.misc.getFixedupDisplayStatus(this.get('status'));
+ }.property('id', 'status'),
+
+ taskAttemptIconStatus: function() {
+ return App.Helpers.misc.getStatusClassForEntity(this.get('taskAttemptStatus'));
}.property('id', 'status', 'counterGroups'),
});
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/tasks_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/tasks_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/tasks_controller.js
index 1bc45b9..35e1fd3 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/tasks_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/tasks_controller.js
@@ -88,9 +88,10 @@ App.TasksController = Em.ObjectController.extend(App.PaginatedContentMixin, App.
{{view.cellContent.status}}</span>')
}),
getCellContent: function(row) {
+ var taskStatus = row.get('status');
return {
- status: row.get('status'),
- statusIcon: App.Helpers.misc.getStatusClassForEntity(row)
+ status: taskStatus,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(taskStatus)
};
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
index 79ae56b..151438e 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
@@ -158,9 +158,10 @@ App.TezAppDagsController = Em.ObjectController.extend(App.PaginatedContentMixin,
if (Ember.typeOf(row.get('progress')) === 'number') {
pct = App.Helpers.number.fractionToPercentage(row.get('progress'));
}
+ var dagStatus = row.get('status');
return {
- status: row.get('status'),
- statusIcon: App.Helpers.misc.getStatusClassForEntity(row),
+ status: dagStatus,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(dagStatus),
progress: pct
};
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
index f6893b9..89a99a9 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
@@ -23,19 +23,19 @@ App.VertexController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
loading: true,
- updateLoading: function() {
+ loadAdditional: function(vertex) {
var loaders = [],
that = this,
- applicationId = this.get('applicationId');
+ applicationId = vertex.get('applicationId');
- if (this.get('status') == 'RUNNING') {
- var vertexIdx = that.get('id').split('_').splice(-1).pop();
+ if (vertex.get('status') == 'RUNNING') {
+ var vertexIdx = vertex.get('id').split('_').splice(-1).pop();
var progressLoader = this.store.find('vertexProgress', vertexIdx, {
appId: applicationId,
- dagIdx: that.get('dagIdx')
+ dagIdx: vertex.get('dagIdx')
}).then(function(vertexProgressInfo) {
if (vertexProgressInfo) {
- that.set('progress', vertexProgressInfo.get('progress'));
+ vertex.set('progress', vertexProgressInfo.get('progress'));
}
}).catch(function(error) {
Em.Logger.error("Failed to fetch vertexProgress" + error)
@@ -46,16 +46,18 @@ App.VertexController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
var appDetailFetcher = that.store.find('appDetail', applicationId).then(function(appDetail) {
var appState = appDetail.get('appState');
if (appState) {
- that.set('yarnAppState', appState);
+ vertex.set('yarnAppState', appState);
}
- that.set('status', App.Helpers.misc.getRealStatus(that.get('status'), appDetail.get('appState'),
+ vertex.set('status', App.Helpers.misc.getRealStatus(vertex.get('status'), appDetail.get('appState'),
appDetail.get('finalAppStatus')));
- });
+ }).catch(function(){});
loaders.push(appDetailFetcher);
Em.RSVP.allSettled(loaders).then(function(){
that.set('loading', false);
});
- }.observes('content'),
+
+ return Em.RSVP.all(loaders);
+ },
childDisplayViews: [
Ember.Object.create({title: 'Details', linkTo: 'vertex.index'}),
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/vertex_index_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_index_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_index_controller.js
index 8c53db2..f175ed1 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_index_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_index_controller.js
@@ -21,7 +21,7 @@ App.VertexIndexController = Em.ObjectController.extend({
//TODO: TEZ-1705 : Create a parent class and move this function there to avoid duplication.
iconStatus: function() {
- return App.Helpers.misc.getStatusClassForEntity(this.get('model'));
+ return App.Helpers.misc.getStatusClassForEntity(this.get('model.status'));
}.property('id', 'status', 'counterGroups'),
progressStr: function() {
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/vertex_task_attempts_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_task_attempts_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_task_attempts_controller.js
index e6ee417..5f51de4 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_task_attempts_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_task_attempts_controller.js
@@ -54,7 +54,8 @@ App.VertexTaskAttemptsController = Em.ObjectController.extend(App.PaginatedConte
},
defaultColumnConfigs: function() {
- var that = this;
+ var that = this,
+ vertexStatus = that.get('controllers.vertex.status');
return [
{
id: 'taskId',
@@ -118,7 +119,7 @@ App.VertexTaskAttemptsController = Em.ObjectController.extend(App.PaginatedConte
headerCellName: 'Status',
filterID: 'status_filter',
filterType: 'dropdown',
- dropdownValues: App.Helpers.misc.taskStatusUIOptions,
+ dropdownValues: App.Helpers.misc.taskAttemptStatusUIOptions,
tableCellViewClass: Em.Table.TableCell.extend({
template: Em.Handlebars.compile(
'<span class="ember-table-content"> \
@@ -126,9 +127,14 @@ App.VertexTaskAttemptsController = Em.ObjectController.extend(App.PaginatedConte
{{view.cellContent.status}}</span>')
}),
getCellContent: function(row) {
+ var status = App.Helpers.misc.getFixedupDisplayStatus(row.get('status'));
+ if (status == 'RUNNING' &&
+ App.Helpers.misc.isStatusInUnsuccessful(vertexStatus)) {
+ status = 'KILLED'
+ }
return {
- status: row.get('status'),
- statusIcon: App.Helpers.misc.getStatusClassForEntity(row)
+ status: status,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(status)
};
}
},
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/controllers/vertex_tasks_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_tasks_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_tasks_controller.js
index 90340ff..9b76aee 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_tasks_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_tasks_controller.js
@@ -50,8 +50,18 @@ App.VertexTasksController = Em.ObjectController.extend(App.PaginatedContentMixin
store.unloadAll(childEntityType);
store.findQuery(childEntityType, this.getFilterProperties()).then(function(entities){
- var pivotLoaders = [];
+ var pivotLoaders = [],
+ vertexStatus = that.get('controllers.vertex.status');
entities.forEach(function (task) {
+ var taskStatus = App.Helpers.misc
+ .getFixedupDisplayStatus(task.get('status'));
+ if (taskStatus == 'RUNNING' &&
+ App.Helpers.misc.isStatusInUnsuccessful(vertexStatus)) {
+ taskStatus = 'KILLED';
+ }
+ if (taskStatus != task.get('status')) {
+ task.set('status', taskStatus);
+ }
var taskAttemptId = task.get('successfulAttemptId') || task.get('attempts.lastObject');
if (!!taskAttemptId) {
// Pivot attempt selection logic
@@ -143,9 +153,10 @@ App.VertexTasksController = Em.ObjectController.extend(App.PaginatedContentMixin
{{view.cellContent.status}}</span>')
}),
getCellContent: function(row) {
+ var status = row.get('status');
return {
- status: row.get('status'),
- statusIcon: App.Helpers.misc.getStatusClassForEntity(row)
+ status: status,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(status)
};
}
},
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
index 4872881..187f800 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
@@ -16,11 +16,10 @@
*/
App.Helpers.misc = {
- getStatusClassForEntity: function(dag) {
- if(!dag) return '';
+ getStatusClassForEntity: function(status, hasFailedTasks) {
+ if(!status) return '';
- var st = dag.get('status');
- switch(st) {
+ switch(status) {
case 'FAILED':
return 'failed';
case 'KILLED':
@@ -30,6 +29,11 @@ App.Helpers.misc = {
case 'ERROR':
return 'error';
case 'SUCCEEDED':
+ if (!!hasFailedTasks) {
+ return 'warning';
+ }
+ /*
+ TODO: TEZ-2113
var counterGroups = dag.get('counterGroups');
var numFailedTasks = this.getCounterValueForDag(counterGroups,
dag.get('id'), 'org.apache.tez.common.counters.DAGCounter',
@@ -38,7 +42,7 @@ App.Helpers.misc = {
if (numFailedTasks > 0) {
return 'warning';
- }
+ }*/
return 'success';
case 'UNDEFINED':
@@ -193,6 +197,21 @@ App.Helpers.misc = {
return path;
},
+ // Tez originally shows the status for task and task attempt only on
+ // completion. this causes confusion to the user as the status would not be
+ // displayed. so if status is not set return the status as 'RUNNING'. We do
+ // not diffentiate between running and scheduled.
+ getFixedupDisplayStatus: function(originalStatus) {
+ // if status is not set show it as running, since originally the task did
+ // not have a status set on scheduled/running.
+ // with the new version we set the status of task as scheduled and that of
+ // task attempt as running
+ if (!originalStatus || originalStatus == 'SCHEDULED') {
+ originalStatus = 'RUNNING';
+ }
+ return originalStatus;
+ },
+
/**
* Merge content of obj2 into obj2, array elements will be concated.
* @param obj1 {Object}
@@ -233,6 +252,14 @@ App.Helpers.misc = {
taskStatusUIOptions: [
{ label: 'All', id: null },
+ { label: 'Running', id: 'SCHEDULED' },
+ { label: 'Succeeded', id: 'SUCCEEDED' },
+ { label: 'Failed', id: 'FAILED' },
+ { label: 'Killed', id: 'KILLED' },
+ ],
+
+ taskAttemptStatusUIOptions: [
+ { label: 'All', id: null },
{ label: 'Running', id: 'RUNNING' },
{ label: 'Succeeded', id: 'SUCCEEDED' },
{ label: 'Failed', id: 'FAILED' },
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/scripts/router.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/router.js b/tez-ui/src/main/webapp/app/scripts/router.js
index efb52d7..0af1d05 100644
--- a/tez-ui/src/main/webapp/app/scripts/router.js
+++ b/tez-ui/src/main/webapp/app/scripts/router.js
@@ -187,6 +187,9 @@ App.VertexRoute = Em.Route.extend({
model: function(params) {
return this.store.find('vertex', params.vertex_id);
},
+ afterModel: function(model) {
+ return this.controllerFor('vertex').loadAdditional(model);
+ },
setupController: setupControllerFactory('Vertex: %@ (%@)', 'name', 'id')
});
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/templates/task/index.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/task/index.hbs b/tez-ui/src/main/webapp/app/templates/task/index.hbs
index df4a5b1..6826e7b 100644
--- a/tez-ui/src/main/webapp/app/templates/task/index.hbs
+++ b/tez-ui/src/main/webapp/app/templates/task/index.hbs
@@ -23,7 +23,7 @@
<tr>
<td>{{t 'common.status'}}</td>
<td>
- {{status}}
+ {{taskStatus}}
<i {{bind-attr class=':task-status taskIconStatus'}}></i>
</td>
</tr>
http://git-wip-us.apache.org/repos/asf/tez/blob/ec7ad206/tez-ui/src/main/webapp/app/templates/taskAttempt/index.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/taskAttempt/index.hbs b/tez-ui/src/main/webapp/app/templates/taskAttempt/index.hbs
index 92e0f13..738b636 100644
--- a/tez-ui/src/main/webapp/app/templates/taskAttempt/index.hbs
+++ b/tez-ui/src/main/webapp/app/templates/taskAttempt/index.hbs
@@ -23,8 +23,8 @@
<tr>
<td>{{t 'common.status'}}</td>
<td>
- {{status}}
- <i {{bind-attr class=':task-status taskIconStatus'}}></i>
+ {{taskAttemptStatus}}
+ <i {{bind-attr class=':task-status taskAttemptIconStatus'}}></i>
</td>
</tr>
<tr>
[17/23] tez git commit: TEZ-2058. Flaky test:
TestTezJobs::testInvalidQueueSubmission. (hitesh)
Posted by ss...@apache.org.
TEZ-2058. Flaky test: TestTezJobs::testInvalidQueueSubmission. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/88d23256
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/88d23256
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/88d23256
Branch: refs/heads/TEZ-2003
Commit: 88d23256ff7e5038ceb4b68877a41d2279b1cbeb
Parents: 0487e0a
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Feb 18 11:10:55 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed Feb 18 11:10:55 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/tez/test/TestTezJobs.java | 43 +++++++++++++++++++-
2 files changed, 42 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/88d23256/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2c643b0..27509d1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -209,6 +209,7 @@ TEZ-UI CHANGES (TEZ-8):
Release 0.5.4: Unreleased
ALL CHANGES:
+ TEZ-2058. Flaky test: TestTezJobs::testInvalidQueueSubmission.
TEZ-2037. Should log TaskAttemptFinishedEvent if taskattempt is recovered to KILLED.
TEZ-2071. TestAMRecovery should set test names for test DAGs.
TEZ-1928. Tez local mode hang in Pig tez local mode.
http://git-wip-us.apache.org/repos/asf/tez/blob/88d23256/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
index b9cb491..b2c9044 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
@@ -58,6 +58,7 @@ import org.apache.tez.dag.api.DataSourceDescriptor;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.InputInitializerDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.SessionNotRunning;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.Vertex;
@@ -694,10 +695,47 @@ public class TestTezJobs {
remoteFs.mkdirs(inputDir);
String outputDirStr = "/tmp/owc-output";
outputPaths[0] = outputDirStr;
+ int result = job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
+ StringUtils.join(",", outputPaths), "2" }, null);
+ Assert.assertTrue("Job should have failed", result != 0);
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("Failed to submit application to YARN"));
+ } finally {
+ if (yarnClient != null) {
+ yarnClient.stop();
+ }
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testInvalidQueueSubmissionToSession() throws Exception {
+
+ TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
+ YarnClient yarnClient = YarnClient.createYarnClient();
+ try {
+
+ yarnClient.init(mrrTezCluster.getConfig());
+ yarnClient.start();
+
+ SimpleSessionExample job = new SimpleSessionExample();
+ tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
+ tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent");
+
+ String[] inputPaths = new String[1];
+ String[] outputPaths = new String[1];
+ String inputDirStr = "/tmp/owc-input";
+ inputPaths[0] = inputDirStr;
+ Path inputDir = new Path(inputDirStr);
+ remoteFs.mkdirs(inputDir);
+ String outputDirStr = "/tmp/owc-output";
+ outputPaths[0] = outputDirStr;
job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
StringUtils.join(",", outputPaths), "2" }, null);
- fail("Job submission should have thrown an exception");
- } catch(IOException e) {
+ fail("Job submission should have failed");
+ } catch (SessionNotRunning e) {
+ // Expected
+ LOG.info("Session not running", e);
+ } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("Failed to submit application to YARN"));
} finally {
if (yarnClient != null) {
@@ -707,6 +745,7 @@ public class TestTezJobs {
}
+
@Test (timeout=60000)
public void testVertexOrder() throws Exception {
TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
[19/23] tez git commit: TEZ-2019. Temporarily allow the scheduler and
launcher to be specified via configuration. (sseth)
Posted by ss...@apache.org.
TEZ-2019. Temporarily allow the scheduler and launcher to be specified
via configuration. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ee954577
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ee954577
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ee954577
Branch: refs/heads/TEZ-2003
Commit: ee95457755e04cf546f761467c9091332ed05442
Parents: 269905b
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Jan 30 16:02:32 2015 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Feb 18 14:39:25 2015 -0800
----------------------------------------------------------------------
TEZ-2003-CHANGES.txt | 4 +++
.../apache/tez/dag/api/TezConfiguration.java | 6 ++++
.../org/apache/tez/dag/app/DAGAppMaster.java | 30 ++++++++++++++++-
.../dag/app/rm/TaskSchedulerEventHandler.java | 34 ++++++++++++++++++--
.../org/apache/tez/runtime/task/TezChild.java | 3 +-
5 files changed, 73 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/ee954577/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
new file mode 100644
index 0000000..1822fcb
--- /dev/null
+++ b/TEZ-2003-CHANGES.txt
@@ -0,0 +1,4 @@
+ALL CHANGES:
+ TEZ-2019. Temporarily allow the scheduler and launcher to be specified via configuration.
+
+INCOMPATIBLE CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/ee954577/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 0bf78f9..c35a853 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1136,6 +1136,12 @@ public class TezConfiguration extends Configuration {
+ "tez-ui.webservice.enable";
public static final boolean TEZ_AM_WEBSERVICE_ENABLE_DEFAULT = true;
+ @ConfigurationScope(Scope.VERTEX)
+ public static final String TEZ_AM_CONTAINER_LAUNCHER_CLASS = TEZ_AM_PREFIX + "container-launcher.class";
+ @ConfigurationScope(Scope.VERTEX)
+ public static final String TEZ_AM_TASK_SCHEDULER_CLASS = TEZ_AM_PREFIX + "task-scheduler.class";
+
+
// TODO only validate property here, value can also be validated if necessary
public static void validateProperty(String property, Scope usedScope) {
Scope validScope = PropertyScope.get(property);
http://git-wip-us.apache.org/repos/asf/tez/blob/ee954577/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 8fd5626..9f523ac 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -27,6 +27,8 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@@ -91,6 +93,7 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.tez.common.AsyncDispatcher;
+import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezConverterUtils;
import org.apache.tez.common.TezUtilsInternal;
@@ -938,9 +941,34 @@ public class DAGAppMaster extends AbstractService {
protected ContainerLauncher
createContainerLauncher(final AppContext context) throws UnknownHostException {
if(isLocal){
+ LOG.info("Creating LocalContainerLauncher");
return new LocalContainerLauncher(context, taskAttemptListener, workingDirectory);
} else {
- return new ContainerLauncherImpl(context);
+ // TODO: Temporary reflection with specific parameters until a clean interface is defined.
+ String containerLauncherClassName = getConfig().get(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHER_CLASS);
+ if (containerLauncherClassName == null) {
+ LOG.info("Creating Default Container Launcher");
+ return new ContainerLauncherImpl(context);
+ } else {
+ LOG.info("Creating container launcher : " + containerLauncherClassName);
+ Class<? extends ContainerLauncher> containerLauncherClazz = (Class<? extends ContainerLauncher>) ReflectionUtils.getClazz(
+ containerLauncherClassName);
+ try {
+ Constructor<? extends ContainerLauncher> ctor = containerLauncherClazz
+ .getConstructor(AppContext.class, Configuration.class, TaskAttemptListener.class);
+ ctor.setAccessible(true);
+ ContainerLauncher instance = ctor.newInstance(context, getConfig(), taskAttemptListener);
+ return instance;
+ } catch (NoSuchMethodException e) {
+ throw new TezUncheckedException(e);
+ } catch (InvocationTargetException e) {
+ throw new TezUncheckedException(e);
+ } catch (InstantiationException e) {
+ throw new TezUncheckedException(e);
+ } catch (IllegalAccessException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ee954577/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 05cbc66..97bd7c8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -18,6 +18,8 @@
package org.apache.tez.dag.app.rm;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.List;
@@ -42,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.TaskLocationHint;
@@ -329,12 +332,39 @@ public class TaskSchedulerEventHandler extends AbstractService
boolean isLocal = getConfig().getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
if (isLocal) {
+ LOG.info("Using TaskScheduler: LocalTaskSchedulerService");
return new LocalTaskSchedulerService(this, this.containerSignatureMatcher,
host, port, trackingUrl, appContext);
}
else {
- return new YarnTaskSchedulerService(this, this.containerSignatureMatcher,
- host, port, trackingUrl, appContext);
+ String schedulerClassName = getConfig().get(TezConfiguration.TEZ_AM_TASK_SCHEDULER_CLASS);
+ if (schedulerClassName == null) {
+ LOG.info("Using TaskScheduler: YarnTaskSchedulerService");
+ return new YarnTaskSchedulerService(this, this.containerSignatureMatcher,
+ host, port, trackingUrl, appContext);
+ } else {
+ LOG.info("Using custom TaskScheduler: " + schedulerClassName);
+ // TODO Temporary reflection with specific parameters. Remove once there is a clean interface.
+ Class<? extends TaskSchedulerService> taskSchedulerClazz =
+ (Class<? extends TaskSchedulerService>) ReflectionUtils.getClazz(schedulerClassName);
+ try {
+ Constructor<? extends TaskSchedulerService> ctor = taskSchedulerClazz
+ .getConstructor(TaskSchedulerAppCallback.class, AppContext.class, String.class,
+ Integer.class, String.class, Configuration.class);
+ ctor.setAccessible(true);
+ TaskSchedulerService taskSchedulerService =
+ ctor.newInstance(this, appContext, host, port, trackingUrl, getConfig());
+ return taskSchedulerService;
+ } catch (NoSuchMethodException e) {
+ throw new TezUncheckedException(e);
+ } catch (InvocationTargetException e) {
+ throw new TezUncheckedException(e);
+ } catch (InstantiationException e) {
+ throw new TezUncheckedException(e);
+ } catch (IllegalAccessException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ee954577/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index d537846..6164e52 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -356,7 +356,8 @@ public class TezChild {
DefaultMetricsSystem.shutdown();
if (!isLocal) {
RPC.stopProxy(umbilical);
- LogManager.shutdown();
+ // TODO Temporary change. Revert. Ideally, move this over to the main method in TezChild if possible.
+// LogManager.shutdown();
}
}
}
[04/23] tez git commit: TEZ-2098. Tez UI: Dag details should be the
default page for dag,
fix invalid time entries for failed Vertices (pramachandran via rbalamohan)
Posted by ss...@apache.org.
TEZ-2098. Tez UI: Dag details should be the default page for dag, fix invalid time entries for failed Vertices (pramachandran via rbalamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b26fdfe7
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b26fdfe7
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b26fdfe7
Branch: refs/heads/TEZ-2003
Commit: b26fdfe7f63553f3c0abc14e30547aebb049f16b
Parents: b74bab4
Author: Rajesh Balamohan <rb...@hortonworks.com>
Authored: Fri Feb 13 15:39:40 2015 +0530
Committer: Rajesh Balamohan <rb...@hortonworks.com>
Committed: Fri Feb 13 15:39:40 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../src/main/webapp/app/scripts/controllers/dags_controller.js | 2 +-
tez-ui/src/main/webapp/app/scripts/helpers/date.js | 6 +++---
3 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b26fdfe7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1e8c116..d62308b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -61,6 +61,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2098. Tez UI: Dag details should be the default page for dag, fix invalid time entries for failed Vertices.
TEZ-2024. TaskFinishedEvent may not be logged in recovery.
TEZ-2031. Tez UI: horizontal scrollbars do not appear in tables, causing them to look truncated.
TEZ-2073. SimpleHistoryLoggingService cannot be read by log aggregation (umask)
http://git-wip-us.apache.org/repos/asf/tez/blob/b26fdfe7/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
index 9d1a243..fbe49ef 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
@@ -137,7 +137,7 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
filterID: 'dagName_filter',
tableCellViewClass: Em.Table.TableCell.extend({
template: Em.Handlebars.compile(
- "{{#link-to 'dag.view' view.cellContent.id class='ember-table-content'}}{{view.cellContent.name}}{{/link-to}}")
+ "{{#link-to 'dag.index' view.cellContent.id class='ember-table-content'}}{{view.cellContent.name}}{{/link-to}}")
}),
getCellContent: function(row) {
return {
http://git-wip-us.apache.org/repos/asf/tez/blob/b26fdfe7/tez-ui/src/main/webapp/app/scripts/helpers/date.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/date.js b/tez-ui/src/main/webapp/app/scripts/helpers/date.js
index 750b4f4..10df10b 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/date.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/date.js
@@ -52,8 +52,8 @@ App.Helpers.date = {
* @method dateFormat
*/
dateFormat: function (timestamp, showSeconds, showMilliseconds) {
- if (!App.Helpers.number.isValidInt(timestamp)) {
- return timestamp;
+ if (!App.Helpers.number.isValidInt(timestamp) || timestamp == 0) {
+ return "";
}
if(showSeconds == undefined) showSeconds = true;
var format = 'DD MMM YYYY HH:mm';
@@ -165,7 +165,7 @@ App.Helpers.date = {
*/
timingFormat: function (time, /* optional */ zeroValid) {
var intTime = parseInt(time);
- if (zeroValid && intTime == 0) {
+ if (zeroValid && intTime <= 0) {
return 0 + ' secs';
}
if (!intTime) {
[11/23] tez git commit: TEZ-2038. TEZ-UI DAG is always running in
tez-ui when the app is failed but no DAGFinishedEvent is logged (Prakash
Ramachandran via zjffdu)
Posted by ss...@apache.org.
TEZ-2038. TEZ-UI DAG is always running in tez-ui when the app is failed but no DAGFinishedEvent is logged (Prakash Ramachandran via zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b6cb7f81
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b6cb7f81
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b6cb7f81
Branch: refs/heads/TEZ-2003
Commit: b6cb7f81c8944e2948195c1fbb9b95faaa257f06
Parents: a5a5665
Author: Jeff Zhang <zj...@apache.org>
Authored: Mon Feb 16 20:03:18 2015 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Mon Feb 16 20:03:18 2015 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../app/scripts/controllers/dag_controller.js | 34 +++++++++++---------
.../app/scripts/controllers/dag_vertices.js | 9 ++++++
.../app/scripts/controllers/dags_controller.js | 4 +++
.../controllers/tez-app-dags-controller.js | 19 +++++++++--
.../scripts/controllers/vertex_controller.js | 2 ++
.../src/main/webapp/app/scripts/helpers/misc.js | 18 +++++++++++
tez-ui/src/main/webapp/app/scripts/router.js | 3 ++
tez-ui/src/main/webapp/app/styles/colors.less | 3 +-
tez-ui/src/main/webapp/app/styles/main.less | 7 +++-
10 files changed, 80 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b6cb7f81/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fd20d20..6638e55 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -64,6 +64,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2038. TEZ-UI DAG is always running in tez-ui when the app is failed but no DAGFinishedEvent is logged.
TEZ-2102. Tez UI: DAG view has hidden edges, dragging DAG by holding vertex causes unintended click.
TEZ-2101. Tez UI: Issues on displaying a table.
TEZ-2092. Tez UI history url handler injects spurious trailing slash.
http://git-wip-us.apache.org/repos/asf/tez/blob/b6cb7f81/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
index b488841..210e673 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
@@ -21,17 +21,17 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
pageTitle: 'Dag',
loading: true,
- updateLoading: function() {
+ loadAdditional: function(dag) {
var that = this;
var loaders = [];
- var applicationId = this.get('applicationId');
- if (this.get('status') === 'RUNNING') {
+ var applicationId = dag.get('applicationId');
+ if (dag.get('status') === 'RUNNING') {
// update the progress info if available. this need not block the UI
- var aminfoLoader = that.store.find('dagProgress', that.get('id'), {
+ var aminfoLoader = that.store.find('dagProgress', dag.get('id'), {
appId: applicationId,
- dagIdx: that.get('idx')
+ dagIdx: dag.get('idx')
}).then(function(dagProgressInfo) {
- that.set('progress', dagProgressInfo.get('progress'));
+ dag.set('progress', dagProgressInfo.get('progress'));
}).catch(function (error) {
Em.Logger.error("Failed to fetch dagProgress")
});
@@ -39,24 +39,26 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
}
var appDetailLoader = this.store.find('appDetail', applicationId)
.then(function(app){
- that.set('appDetail', app);
+ dag.set('appDetail', app);
+ var appState = app.get('appState');
+ if (appState) {
+ dag.set('yarnAppState', appState);
+ }
+ dag.set('status', App.Helpers.misc.getRealStatus(dag.get('status'), app.get('appState'), app.get('finalAppStatus')));
});
var tezAppLoader = this.store.find('tezApp', 'tez_' + applicationId)
.then(function(app){
- that.set('tezApp', app);
+ dag.set('tezApp', app);
});
- var appDetailFetcher = that.store.find('appDetail', applicationId).then(function(appDetail) {
- var appState = appDetail.get('appState');
- if (appState) {
- that.set('yarnAppState', appState);
- }
- });
- loaders.push(appDetailFetcher);
+ loaders.push(appDetailLoader);
+
Em.RSVP.allSettled(loaders).then(function(){
that.set('loading', false);
});
- }.observes('content'),
+
+ return Em.RSVP.all(loaders);
+ },
enableAppIdLink: function() {
return !!(this.get('tezApp') && this.get('appDetail'));
http://git-wip-us.apache.org/repos/asf/tez/blob/b6cb7f81/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
index c26f83e..a373220 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
@@ -49,6 +49,15 @@ App.DagVerticesController = Em.ObjectController.extend(App.PaginatedContentMixin
var that = this;
vertices = this.get('entities');
+
+ var dagStatus = this.get('controllers.dag.status');
+ if (App.Helpers.misc.isStatusInUnsuccessful(dagStatus)) {
+ vertices.filterBy('status', 'RUNNING')
+ .forEach(function(item) {
+ item.set('status', 'KILLED');
+ });
+ }
+
var runningVerticesIdx = vertices
.filterBy('status', 'RUNNING')
.map(function(item) {
http://git-wip-us.apache.org/repos/asf/tez/blob/b6cb7f81/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
index fbe49ef..53d9eeb 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
@@ -79,6 +79,10 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
fetcher = store.find('appDetail', appId);
fetcher.then(function (app) {
dag.set('appDetail', app);
+ if (dag.get('status') === 'RUNNING') {
+ dag.set('status', App.Helpers.misc.getRealStatus(dag.get('status'), app.get('appState'),
+ app.get('finalAppStatus')));
+ }
});
loaders.push(fetcher);
//Load tezApp details
http://git-wip-us.apache.org/repos/asf/tez/blob/b6cb7f81/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
index 438ec29..79ae56b 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/tez-app-dags-controller.js
@@ -57,14 +57,17 @@ App.TezAppDagsController = Em.ObjectController.extend(App.PaginatedContentMixin,
that.set('loading', true);
this.get('store').unloadAll(childEntityType);
- this.get('store').findQuery(childEntityType, this.getFilterProperties()).then(function(entities){
+ this.get('store').findQuery(childEntityType, this.getFilterProperties())
+ .then(function(entities){
+
that.set('entities', entities);
var loaders = [];
entities.forEach(function (dag) {
+ var applicationId = dag.get('applicationId');
if (dag.get('status') === 'RUNNING') {
amInfoFetcher = that.store.find('dagProgress', dag.get('id'), {
- appId: dag.get('applicationId'),
+ appId: applicationId,
dagIdx: dag.get('idx')
}).then(function(dagProgressInfo) {
dag.set('progress', dagProgressInfo.get('progress'));
@@ -73,7 +76,19 @@ App.TezAppDagsController = Em.ObjectController.extend(App.PaginatedContentMixin,
});
loaders.push(amInfoFetcher);
}
+
+ var appDetailLoader = that.store.find('appDetail', applicationId)
+ .then(function(app){
+ dag.set('appDetail', app);
+ var appState = app.get('appState');
+ if (appState) {
+ dag.set('yarnAppState', appState);
+ }
+ dag.set('status', App.Helpers.misc.getRealStatus(dag.get('status'),
+ app.get('appState'), app.get('finalAppStatus')));
+ });
});
+
Em.RSVP.allSettled(loaders).then(function(){
that.set('loading', false);
});
http://git-wip-us.apache.org/repos/asf/tez/blob/b6cb7f81/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
index 3c56690..f6893b9 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/vertex_controller.js
@@ -48,6 +48,8 @@ App.VertexController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
if (appState) {
that.set('yarnAppState', appState);
}
+ that.set('status', App.Helpers.misc.getRealStatus(that.get('status'), appDetail.get('appState'),
+ appDetail.get('finalAppStatus')));
});
loaders.push(appDetailFetcher);
Em.RSVP.allSettled(loaders).then(function(){
http://git-wip-us.apache.org/repos/asf/tez/blob/b6cb7f81/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
index 50663a8..4872881 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
@@ -41,11 +41,25 @@ App.Helpers.misc = {
}
return 'success';
+ case 'UNDEFINED':
+ return 'unknown';
default:
return 'submitted';
}
},
+ getRealStatus: function(entityState, yarnAppState, yarnAppFinalState) {
+ if (entityState != 'RUNNING' || (yarnAppState != 'FINISHED' && yarnAppState != 'KILLED' && yarnAppState != 'FAILED')) {
+ return entityState;
+ }
+
+ if (yarnAppState == 'KILLED' || yarnAppState == 'FAILED') {
+ return yarnAppState;
+ }
+
+ return yarnFinalState;
+ },
+
getCounterValueForDag: function(counterGroups, dagID, counterGroupName, counterName) {
if (!counterGroups) {
return 0;
@@ -76,6 +90,10 @@ App.Helpers.misc = {
return $.inArray(status, ['RUNNING', 'SUCCEEDED', 'FAILED', 'KILLED']) != -1;
},
+ isStatusInUnsuccessful: function(status) {
+ return $.inArray(status, ['FAILED', 'KILLED', 'UNDEFINED']) != -1;
+ },
+
/**
* To trim a complete class path with namespace to the class name.
*/
http://git-wip-us.apache.org/repos/asf/tez/blob/b6cb7f81/tez-ui/src/main/webapp/app/scripts/router.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/router.js b/tez-ui/src/main/webapp/app/scripts/router.js
index b1f750b..efb52d7 100644
--- a/tez-ui/src/main/webapp/app/scripts/router.js
+++ b/tez-ui/src/main/webapp/app/scripts/router.js
@@ -141,6 +141,9 @@ App.DagRoute = Em.Route.extend({
model: function(params) {
return this.store.find('dag', params.dag_id);
},
+ afterModel: function(model) {
+ return this.controllerFor('dag').loadAdditional(model);
+ },
setupController: setupControllerFactory('Dag: %@ (%@)', 'name', 'id')
});
http://git-wip-us.apache.org/repos/asf/tez/blob/b6cb7f81/tez-ui/src/main/webapp/app/styles/colors.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/colors.less b/tez-ui/src/main/webapp/app/styles/colors.less
index b8e3b6e..0fee7c0 100644
--- a/tez-ui/src/main/webapp/app/styles/colors.less
+++ b/tez-ui/src/main/webapp/app/styles/colors.less
@@ -33,4 +33,5 @@
@success-color: limegreen;
@error-color: crimson;
-@warning-color: orange;
\ No newline at end of file
+@warning-color: orange;
+@unknown-color: crimson;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/b6cb7f81/tez-ui/src/main/webapp/app/styles/main.less
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/styles/main.less b/tez-ui/src/main/webapp/app/styles/main.less
index 161c8f3..fef8c00 100644
--- a/tez-ui/src/main/webapp/app/styles/main.less
+++ b/tez-ui/src/main/webapp/app/styles/main.less
@@ -338,13 +338,18 @@ body, html, body > .ember-view {
&.killed {
.fa-icon(exclamation-circle);
- color: @warning-color;
+ color: @error-color;
}
&.warning {
.fa-icon(exclamation-triangle);
color: @warning-color;
}
+
+ &.unknown {
+ .fa-icon(exclamation);
+ color: @unknown-color;
+ }
}
.fa-action {
[02/23] tez git commit: TEZ-2037. Should log TaskAttemptFinishedEvent
if taskattempt is recovered to KILLED (zjffdu)
Posted by ss...@apache.org.
TEZ-2037. Should log TaskAttemptFinishedEvent if taskattempt is recovered to KILLED (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c266289c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c266289c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c266289c
Branch: refs/heads/TEZ-2003
Commit: c266289cde6693e8a586e3c4b2bdffbd6b98b9ca
Parents: bc6a490
Author: Jeff Zhang <zj...@apache.org>
Authored: Fri Feb 13 10:01:33 2015 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Fri Feb 13 10:01:33 2015 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 1 +
.../app/dag/impl/TestTaskAttemptRecovery.java | 66 +++++++++++++++++++-
3 files changed, 66 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/c266289c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b33c454..91183eb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -195,6 +195,7 @@ TEZ-UI CHANGES (TEZ-8):
Release 0.5.4: Unreleased
ALL CHANGES:
+ TEZ-2037. Should log TaskAttemptFinishedEvent if taskattempt is recovered to KILLED.
TEZ-2071. TestAMRecovery should set test names for test DAGs.
TEZ-1928. Tez local mode hang in Pig tez local mode.
TEZ-1893. Verify invalid -1 parallelism in DAG.verify().
http://git-wip-us.apache.org/repos/asf/tez/blob/c266289c/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index ccac620..4a4506e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -1392,6 +1392,7 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskEventType.T_ATTEMPT_KILLED));
taskAttempt.sendEvent(createDAGCounterUpdateEventTAFinished(taskAttempt,
getExternalState(TaskAttemptStateInternal.KILLED)));
+ taskAttempt.logJobHistoryAttemptUnsuccesfulCompletion(TaskAttemptState.KILLED);
endState = TaskAttemptStateInternal.KILLED;
break;
case SUCCEEDED:
http://git-wip-us.apache.org/repos/asf/tez/blob/c266289c/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
index e5fcd72..9d0e121 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
@@ -20,10 +20,15 @@ package org.apache.tez.dag.app.dag.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -39,16 +44,22 @@ import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
+import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttemptStateInternal;
+import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.event.DAGEventCounterUpdate;
import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
import org.apache.tez.dag.app.dag.event.TaskEventTAUpdate;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.HistoryEventHandler;
+import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.history.events.TaskAttemptFinishedEvent;
import org.apache.tez.dag.history.events.TaskAttemptStartedEvent;
import org.apache.tez.dag.records.TaskAttemptTerminationCause;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
@@ -61,11 +72,59 @@ public class TestTaskAttemptRecovery {
private long startTime = System.currentTimeMillis();
private long finishTime = startTime + 5000;
- private TezTaskAttemptID taId = mock(TezTaskAttemptID.class);
+ private TezTaskAttemptID taId;
private String vertexName = "v1";
+ private AppContext mockAppContext;
+ private MockHistoryEventHandler mockHistoryEventHandler;
+ private Task mockTask;
+ private Vertex mockVertex;
+
+ public static class MockHistoryEventHandler extends HistoryEventHandler {
+
+ private List<DAGHistoryEvent> events;
+
+ public MockHistoryEventHandler(AppContext context) {
+ super(context);
+ events = new ArrayList<DAGHistoryEvent>();
+ }
+
+ @Override
+ public void handle(DAGHistoryEvent event) {
+ events.add(event);
+ }
+
+ @Override
+ public void handleCriticalEvent(DAGHistoryEvent event) throws IOException {
+ events.add(event);
+ }
+
+ void verfiyTaskAttemptFinishedEvent(TezTaskAttemptID taId, TaskAttemptState finalState, int expectedTimes) {
+ int actualTimes = 0;
+ for (DAGHistoryEvent event : events) {
+ if (event.getHistoryEvent().getEventType() == HistoryEventType.TASK_ATTEMPT_FINISHED) {
+ TaskAttemptFinishedEvent tfEvent = (TaskAttemptFinishedEvent)event.getHistoryEvent();
+ if (tfEvent.getTaskAttemptID().equals(taId) &&
+ tfEvent.getState().equals(finalState)) {
+ actualTimes ++;
+ }
+ }
+ }
+ assertEquals(expectedTimes, actualTimes);
+ }
+ }
+
@Before
public void setUp() {
+ mockTask = mock(Task.class);
+ mockVertex = mock(Vertex.class);
+ when(mockTask.getVertex()).thenReturn(mockVertex);
+ mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
+ when(mockAppContext.getCurrentDAG().getVertex(any(TezVertexID.class))
+ .getTask(any(TezTaskID.class)))
+ .thenReturn(mockTask);
+ mockHistoryEventHandler = new MockHistoryEventHandler(mockAppContext);
+ when(mockAppContext.getHistoryHandler()).thenReturn(mockHistoryEventHandler);
mockEventHandler = mock(EventHandler.class);
TezTaskID taskId =
TezTaskID.fromString("task_1407371892933_0001_1_00_000000");
@@ -73,8 +132,9 @@ public class TestTaskAttemptRecovery {
new TaskAttemptImpl(taskId, 0, mockEventHandler,
mock(TaskAttemptListener.class), new Configuration(),
new SystemClock(), mock(TaskHeartbeatHandler.class),
- mock(AppContext.class), false, Resource.newInstance(1, 1),
+ mockAppContext, false, Resource.newInstance(1, 1),
mock(ContainerContext.class), false);
+ taId = ta.getID();
}
private void restoreFromTAStartEvent() {
@@ -157,6 +217,8 @@ public class TestTaskAttemptRecovery {
verifyEvents(events, TaskEventTAUpdate.class, 1);
// one for task launch, one for task killed
verifyEvents(events, DAGEventCounterUpdate.class, 2);
+
+ mockHistoryEventHandler.verfiyTaskAttemptFinishedEvent(taId, TaskAttemptState.KILLED, 1);
}
/**
[09/23] tez git commit: TEZ-2095. master branch fails to compile
against hadoop-2.4. (hitesh)
Posted by ss...@apache.org.
TEZ-2095. master branch fails to compile against hadoop-2.4. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8e138a22
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8e138a22
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8e138a22
Branch: refs/heads/TEZ-2003
Commit: 8e138a22e8aaca3d6e6bc94030b6ddebcf2a37e3
Parents: 48fd4d6
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Feb 13 15:58:39 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Feb 13 15:58:39 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java | 4 ++--
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/8e138a22/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d316290..33f9614 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.7.0: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2095. master branch fails to compile against hadoop-2.4.
TEZ-2093. Add events to MockDAGAppMaster and add e2e test for event routing
TEZ-2075. Incompatible issue caused by TEZ-1233 that TezConfiguration.TEZ_SITE_XML is made private
TEZ-2082. Race condition in TaskAttemptListenerImpTezDag.getTask()
http://git-wip-us.apache.org/repos/asf/tez/blob/8e138a22/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
index 599a289..f0f7dc5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
@@ -156,9 +156,9 @@ public class TestTaskAttemptListenerImplTezDag {
assertNull(containerTask);
}
- private ContainerId createContainerId(ApplicationId applicationId, long containerIdLong) {
+ private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
- ContainerId containerId = ContainerId.newContainerId(appAttemptId, containerIdLong);
+ ContainerId containerId = ContainerId.newInstance(appAttemptId, containerIdx);
return containerId;
}
[15/23] tez git commit: TEZ-2116. Tez UI: dags page filter does not
work if more than one filter is specified. (Prakash Ramachandran via hitesh)
Posted by ss...@apache.org.
TEZ-2116. Tez UI: dags page filter does not work if more than one filter is specified. (Prakash Ramachandran via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b756bc0d
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b756bc0d
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b756bc0d
Branch: refs/heads/TEZ-2003
Commit: b756bc0d406ac519de87d8c088c954e81c188ae6
Parents: dd44bde
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Feb 18 10:11:04 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed Feb 18 10:11:04 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../app/scripts/controllers/dags_controller.js | 4 ++--
.../app/scripts/mixins/paginated_content.js | 21 ++++++++++++++++++--
3 files changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b756bc0d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a566a37..ff3adc8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -64,6 +64,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2116. Tez UI: dags page filter does not work if more than one filter is specified.
TEZ-2106. TEZ UI: Display data load time, and add a refresh button for items that can be refreshed.
TEZ-2114. Tez UI: task/task attempt status is not available when its running.
TEZ-2112. Tez UI: fix offset calculation, add home button to breadcrumbs.
http://git-wip-us.apache.org/repos/asf/tez/blob/b756bc0d/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
index 880378a..c1aea73 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
@@ -50,9 +50,9 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
loadData: function() {
var filters = {
primary: {
- user: this.user_filter,
+ dagName: this.dagName_filter,
applicationId: this.appId_filter,
- dagName: this.dagName_filter
+ user: this.user_filter
},
secondary: {
status: this.status_filter
http://git-wip-us.apache.org/repos/asf/tez/blob/b756bc0d/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js b/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
index 7b08036..f371f28 100644
--- a/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
+++ b/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
@@ -181,11 +181,28 @@ App.PaginatedContentMixin = Em.Mixin.create({
};
var f = this._paginationFilters;
- var primary = f.primary;
+ var primary = f.primary || {};
var secondary = f.secondary || {};
- primary = this._concatFilters(primary);
+ // TimelineRest API allows only one primaryFilter but any number of
+ // secondary filters. secondary filters are first checked in otherInfo
+ // field and then in primaryFilter field. this is tricky (for ex. when
+ // otherInfo and primaryFilter has same key). so we move all filters
+ // other than first non null primary to secondary.
+ var foundOnePrimaryFilter = false;
+ $.each(primary, function(name, value) {
+ if (!value) {
+ delete primary[name];
+ return true;
+ }
+ if (foundOnePrimaryFilter) {
+ secondary[name] = value;
+ delete primary[name];
+ }
+ foundOnePrimaryFilter = true;
+ });
+ primary = this._concatFilters(primary);
secondary = this._concatFilters(secondary);
if (!Em.empty(primary)) {
[21/23] tez git commit: TEZ-2090. Add tests for jobs running in
external services. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
new file mode 100644
index 0000000..a93c1a4
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.tests;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.app.launcher.TezTestServiceNoOpContainerLauncher;
+import org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerService;
+import org.apache.tez.dag.app.taskcomm.TezTestServiceTaskCommunicatorImpl;
+import org.apache.tez.examples.HashJoinExample;
+import org.apache.tez.examples.JoinDataGen;
+import org.apache.tez.examples.JoinValidate;
+import org.apache.tez.service.MiniTezTestServiceCluster;
+import org.apache.tez.test.MiniTezCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestExternalTezServices {
+
+ private static final Log LOG = LogFactory.getLog(TestExternalTezServices.class);
+
+ private static MiniTezCluster tezCluster;
+ private static MiniDFSCluster dfsCluster;
+ private static MiniTezTestServiceCluster tezTestServiceCluster;
+
+ private static Configuration clusterConf = new Configuration();
+ private static Configuration confForJobs;
+
+ private static FileSystem remoteFs;
+ private static FileSystem localFs;
+
+ private static TezClient sharedTezClient;
+
+ private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + TestExternalTezServices.class.getName()
+ + "-tmpDir";
+
+ @BeforeClass
+ public static void setup() throws IOException, TezException, InterruptedException {
+
+ localFs = FileSystem.getLocal(clusterConf);
+
+ try {
+ clusterConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
+ dfsCluster =
+ new MiniDFSCluster.Builder(clusterConf).numDataNodes(1).format(true).racks(null).build();
+ remoteFs = dfsCluster.getFileSystem();
+ LOG.info("MiniDFSCluster started");
+ } catch (IOException io) {
+ throw new RuntimeException("problem starting mini dfs cluster", io);
+ }
+
+ tezCluster = new MiniTezCluster(TestExternalTezServices.class.getName(), 1, 1, 1);
+ Configuration conf = new Configuration();
+ conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
+ tezCluster.init(conf);
+ tezCluster.start();
+ LOG.info("MiniTezCluster started");
+
+ clusterConf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
+ for (Map.Entry<String, String> entry : tezCluster.getConfig()) {
+ clusterConf.set(entry.getKey(), entry.getValue());
+ }
+ long jvmMax = Runtime.getRuntime().maxMemory();
+
+ tezTestServiceCluster = MiniTezTestServiceCluster
+ .create(TestExternalTezServices.class.getSimpleName(), 3, ((long) (jvmMax * 0.5d)), 1);
+ tezTestServiceCluster.init(clusterConf);
+ tezTestServiceCluster.start();
+ LOG.info("MiniTezTestServer started");
+
+ confForJobs = new Configuration(clusterConf);
+ for (Map.Entry<String, String> entry : tezTestServiceCluster
+ .getClusterSpecificConfiguration()) {
+ confForJobs.set(entry.getKey(), entry.getValue());
+ }
+
+ // TODO TEZ-2003 Once per vertex configuration is possible, run separate tests for push vs pull (regular threaded execution)
+
+ Path stagingDirPath = new Path("/tmp/tez-staging-dir");
+ remoteFs.mkdirs(stagingDirPath);
+ // This is currently configured to push tasks into the Service, and then use the standard RPC
+ confForJobs.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
+ confForJobs.set(TezConfiguration.TEZ_AM_TASK_SCHEDULER_CLASS,
+ TezTestServiceTaskSchedulerService.class.getName());
+ confForJobs.set(TezConfiguration.TEZ_AM_CONTAINER_LAUNCHER_CLASS,
+ TezTestServiceNoOpContainerLauncher.class.getName());
+ confForJobs.set(TezConfiguration.TEZ_AM_TASK_COMMUNICATOR_CLASS,
+ TezTestServiceTaskCommunicatorImpl.class.getName());
+
+ TezConfiguration tezConf = new TezConfiguration(confForJobs);
+
+ sharedTezClient = TezClient.create(TestExternalTezServices.class.getSimpleName() + "_session",
+ tezConf, true);
+ sharedTezClient.start();
+ LOG.info("Shared TezSession started");
+ sharedTezClient.waitTillReady();
+ LOG.info("Shared TezSession ready for submission");
+
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException, TezException {
+ if (sharedTezClient != null) {
+ sharedTezClient.stop();
+ sharedTezClient = null;
+ }
+
+ if (tezTestServiceCluster != null) {
+ tezTestServiceCluster.stop();
+ tezTestServiceCluster = null;
+ }
+
+ if (tezCluster != null) {
+ tezCluster.stop();
+ tezCluster = null;
+ }
+ if (dfsCluster != null) {
+ dfsCluster.shutdown();
+ dfsCluster = null;
+ }
+ // TODO Add cleanup code.
+ }
+
+
+ @Test(timeout = 60000)
+ public void test1() throws Exception {
+ Path testDir = new Path("/tmp/testHashJoinExample");
+
+ remoteFs.mkdirs(testDir);
+
+ Path dataPath1 = new Path(testDir, "inPath1");
+ Path dataPath2 = new Path(testDir, "inPath2");
+ Path expectedOutputPath = new Path(testDir, "expectedOutputPath");
+ Path outPath = new Path(testDir, "outPath");
+
+ TezConfiguration tezConf = new TezConfiguration(confForJobs);
+
+ JoinDataGen dataGen = new JoinDataGen();
+ String[] dataGenArgs = new String[]{
+ dataPath1.toString(), "1048576", dataPath2.toString(), "524288",
+ expectedOutputPath.toString(), "2"};
+ assertEquals(0, dataGen.run(tezConf, dataGenArgs, sharedTezClient));
+
+ HashJoinExample joinExample = new HashJoinExample();
+ String[] args = new String[]{
+ dataPath1.toString(), dataPath2.toString(), "2", outPath.toString()};
+ assertEquals(0, joinExample.run(tezConf, args, sharedTezClient));
+
+ JoinValidate joinValidate = new JoinValidate();
+ String[] validateArgs = new String[]{
+ expectedOutputPath.toString(), outPath.toString(), "3"};
+ assertEquals(0, joinValidate.run(tezConf, validateArgs, sharedTezClient));
+
+ // Ensure this was actually submitted to the external cluster
+ assertTrue(tezTestServiceCluster.getNumSubmissions() > 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java b/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java
new file mode 100644
index 0000000..60ebc53
--- /dev/null
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.tez.dag.api.DagTypeConverters;
+import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.OutputDescriptor;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.GroupInputSpec;
+import org.apache.tez.runtime.api.impl.InputSpec;
+import org.apache.tez.runtime.api.impl.OutputSpec;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.GroupInputSpecProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.IOSpecProto;
+import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.TaskSpecProto;
+
+public class ProtoConverters {
+
+ public static TaskSpec getTaskSpecfromProto(TaskSpecProto taskSpecProto) {
+ TezTaskAttemptID taskAttemptID =
+ TezTaskAttemptID.fromString(taskSpecProto.getTaskAttemptIdString());
+
+ ProcessorDescriptor processorDescriptor = null;
+ if (taskSpecProto.hasProcessorDescriptor()) {
+ processorDescriptor = DagTypeConverters
+ .convertProcessorDescriptorFromDAGPlan(taskSpecProto.getProcessorDescriptor());
+ }
+
+ List<InputSpec> inputSpecList = new ArrayList<InputSpec>(taskSpecProto.getInputSpecsCount());
+ if (taskSpecProto.getInputSpecsCount() > 0) {
+ for (IOSpecProto inputSpecProto : taskSpecProto.getInputSpecsList()) {
+ inputSpecList.add(getInputSpecFromProto(inputSpecProto));
+ }
+ }
+
+ List<OutputSpec> outputSpecList =
+ new ArrayList<OutputSpec>(taskSpecProto.getOutputSpecsCount());
+ if (taskSpecProto.getOutputSpecsCount() > 0) {
+ for (IOSpecProto outputSpecProto : taskSpecProto.getOutputSpecsList()) {
+ outputSpecList.add(getOutputSpecFromProto(outputSpecProto));
+ }
+ }
+
+ List<GroupInputSpec> groupInputSpecs =
+ new ArrayList<GroupInputSpec>(taskSpecProto.getGroupedInputSpecsCount());
+ if (taskSpecProto.getGroupedInputSpecsCount() > 0) {
+ for (GroupInputSpecProto groupInputSpecProto : taskSpecProto.getGroupedInputSpecsList()) {
+ groupInputSpecs.add(getGroupInputSpecFromProto(groupInputSpecProto));
+ }
+ }
+
+ TaskSpec taskSpec =
+ new TaskSpec(taskAttemptID, taskSpecProto.getDagName(), taskSpecProto.getVertexName(),
+ taskSpecProto.getVertexParallelism(), processorDescriptor, inputSpecList,
+ outputSpecList, groupInputSpecs);
+ return taskSpec;
+ }
+
+ public static TaskSpecProto convertTaskSpecToProto(TaskSpec taskSpec) {
+ TaskSpecProto.Builder builder = TaskSpecProto.newBuilder();
+ builder.setTaskAttemptIdString(taskSpec.getTaskAttemptID().toString());
+ builder.setDagName(taskSpec.getDAGName());
+ builder.setVertexName(taskSpec.getVertexName());
+ builder.setVertexParallelism(taskSpec.getVertexParallelism());
+
+ if (taskSpec.getProcessorDescriptor() != null) {
+ builder.setProcessorDescriptor(
+ DagTypeConverters.convertToDAGPlan(taskSpec.getProcessorDescriptor()));
+ }
+
+ if (taskSpec.getInputs() != null && !taskSpec.getInputs().isEmpty()) {
+ for (InputSpec inputSpec : taskSpec.getInputs()) {
+ builder.addInputSpecs(convertInputSpecToProto(inputSpec));
+ }
+ }
+
+ if (taskSpec.getOutputs() != null && !taskSpec.getOutputs().isEmpty()) {
+ for (OutputSpec outputSpec : taskSpec.getOutputs()) {
+ builder.addOutputSpecs(convertOutputSpecToProto(outputSpec));
+ }
+ }
+
+ if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) {
+ for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) {
+ builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec));
+
+ }
+ }
+ return builder.build();
+ }
+
+
+ public static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) {
+ InputDescriptor inputDescriptor = null;
+ if (inputSpecProto.hasIoDescriptor()) {
+ inputDescriptor =
+ DagTypeConverters.convertInputDescriptorFromDAGPlan(inputSpecProto.getIoDescriptor());
+ }
+ InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor,
+ inputSpecProto.getPhysicalEdgeCount());
+ return inputSpec;
+ }
+
+ public static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) {
+ IOSpecProto.Builder builder = IOSpecProto.newBuilder();
+ if (inputSpec.getSourceVertexName() != null) {
+ builder.setConnectedVertexName(inputSpec.getSourceVertexName());
+ }
+ if (inputSpec.getInputDescriptor() != null) {
+ builder.setIoDescriptor(DagTypeConverters.convertToDAGPlan(inputSpec.getInputDescriptor()));
+ }
+ builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount());
+ return builder.build();
+ }
+
+ public static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) {
+ OutputDescriptor outputDescriptor = null;
+ if (outputSpecProto.hasIoDescriptor()) {
+ outputDescriptor =
+ DagTypeConverters.convertOutputDescriptorFromDAGPlan(outputSpecProto.getIoDescriptor());
+ }
+ OutputSpec outputSpec =
+ new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor,
+ outputSpecProto.getPhysicalEdgeCount());
+ return outputSpec;
+ }
+
+ public static IOSpecProto convertOutputSpecToProto(OutputSpec outputSpec) {
+ IOSpecProto.Builder builder = IOSpecProto.newBuilder();
+ if (outputSpec.getDestinationVertexName() != null) {
+ builder.setConnectedVertexName(outputSpec.getDestinationVertexName());
+ }
+ if (outputSpec.getOutputDescriptor() != null) {
+ builder.setIoDescriptor(DagTypeConverters.convertToDAGPlan(outputSpec.getOutputDescriptor()));
+ }
+ builder.setPhysicalEdgeCount(outputSpec.getPhysicalEdgeCount());
+ return builder.build();
+ }
+
+ public static GroupInputSpec getGroupInputSpecFromProto(GroupInputSpecProto groupInputSpecProto) {
+ GroupInputSpec groupSpec = new GroupInputSpec(groupInputSpecProto.getGroupName(),
+ groupInputSpecProto.getGroupVerticesList(), DagTypeConverters
+ .convertInputDescriptorFromDAGPlan(groupInputSpecProto.getMergedInputDescriptor()));
+ return groupSpec;
+ }
+
+ public static GroupInputSpecProto convertGroupInputSpecToProto(GroupInputSpec groupInputSpec) {
+ GroupInputSpecProto.Builder builder = GroupInputSpecProto.newBuilder();
+ builder.setGroupName(groupInputSpec.getGroupName());
+ builder.addAllGroupVertices(groupInputSpec.getGroupVertices());
+ builder.setMergedInputDescriptor(
+ DagTypeConverters.convertToDAGPlan(groupInputSpec.getMergedInputDescriptor()));
+ return builder.build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/proto/TezDaemonProtocol.proto
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/proto/TezDaemonProtocol.proto b/tez-ext-service-tests/src/test/proto/TezDaemonProtocol.proto
new file mode 100644
index 0000000..2f8b2e6
--- /dev/null
+++ b/tez-ext-service-tests/src/test/proto/TezDaemonProtocol.proto
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tez.test.service.rpc";
+option java_outer_classname = "TezTestServiceProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+import "DAGApiRecords.proto";
+
+message IOSpecProto {
+ optional string connected_vertex_name = 1;
+ optional TezEntityDescriptorProto io_descriptor = 2;
+ optional int32 physical_edge_count = 3;
+}
+
+message GroupInputSpecProto {
+ optional string group_name = 1;
+ repeated string group_vertices = 2;
+ optional TezEntityDescriptorProto merged_input_descriptor = 3;
+}
+
+message TaskSpecProto {
+ optional string task_attempt_id_string = 1;
+ optional string dag_name = 2;
+ optional string vertex_name = 3;
+ optional TezEntityDescriptorProto processor_descriptor = 4;
+ repeated IOSpecProto input_specs = 5;
+ repeated IOSpecProto output_specs = 6;
+ repeated GroupInputSpecProto grouped_input_specs = 7;
+ optional int32 vertex_parallelism = 8;
+}
+
+
+message SubmitWorkRequestProto {
+ optional string container_id_string = 1;
+ optional string am_host = 2;
+ optional int32 am_port = 3;
+ optional string token_identifier = 4;
+ optional bytes credentials_binary = 5;
+ optional string user = 6;
+ optional string application_id_string = 7;
+ optional int32 app_attempt_number = 8;
+ optional TaskSpecProto task_spec = 9;
+}
+
+message SubmitWorkResponseProto {
+}
+
+
+
+message RunContainerRequestProto {
+ optional string container_id_string = 1;
+ optional string am_host = 2;
+ optional int32 am_port = 3;
+ optional string token_identifier = 4;
+ optional bytes credentials_binary = 5;
+ optional string user = 6;
+ optional string application_id_string = 7;
+ optional int32 app_attempt_number = 8;
+}
+
+message RunContainerResponseProto {
+}
+
+service TezTestServiceProtocol {
+ rpc runContainer(RunContainerRequestProto) returns (RunContainerResponseProto);
+ rpc submitWork(SubmitWorkRequestProto) returns (SubmitWorkResponseProto);
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-ext-service-tests/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/resources/log4j.properties b/tez-ext-service-tests/src/test/resources/log4j.properties
new file mode 100644
index 0000000..531b68b
--- /dev/null
+++ b/tez-ext-service-tests/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 6164e52..30f80a5 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -390,7 +390,7 @@ public class TezChild {
private final Throwable throwable;
private final String errorMessage;
- ContainerExecutionResult(ExitStatus exitStatus, @Nullable Throwable throwable,
+ public ContainerExecutionResult(ExitStatus exitStatus, @Nullable Throwable throwable,
@Nullable String errorMessage) {
this.exitStatus = exitStatus;
this.throwable = throwable;
http://git-wip-us.apache.org/repos/asf/tez/blob/d4134b23/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
index 6606481..34f2fc6 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
@@ -67,7 +67,7 @@ public class TezTaskRunner implements TezUmbilical, ErrorReporter {
private final AtomicBoolean taskRunning;
private final AtomicBoolean shutdownRequested = new AtomicBoolean(false);
- TezTaskRunner(Configuration tezConf, UserGroupInformation ugi, String[] localDirs,
+ public TezTaskRunner(Configuration tezConf, UserGroupInformation ugi, String[] localDirs,
TaskSpec taskSpec, TezTaskUmbilicalProtocol umbilical, int appAttemptNumber,
Map<String, ByteBuffer> serviceConsumerMetadata, Map<String, String> serviceProviderEnvMap,
Multimap<String, String> startedInputsMap, TaskReporter taskReporter,
[03/23] tez git commit: TEZ-2024. TaskFinishedEvent may not be logged
in recovery (zjffdu)
Posted by ss...@apache.org.
TEZ-2024. TaskFinishedEvent may not be logged in recovery (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b74bab47
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b74bab47
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b74bab47
Branch: refs/heads/TEZ-2003
Commit: b74bab471649021cf9e8d4f9087bfce0fa9bf757
Parents: c266289
Author: Jeff Zhang <zj...@apache.org>
Authored: Fri Feb 13 11:06:38 2015 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Fri Feb 13 11:06:38 2015 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/dag/app/dag/impl/TaskImpl.java | 1 +
.../dag/app/dag/impl/TestTaskAttemptRecovery.java | 15 +++++++++++++++
.../tez/dag/app/dag/impl/TestTaskRecovery.java | 7 ++++++-
4 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b74bab47/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 91183eb..1e8c116 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -61,6 +61,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2024. TaskFinishedEvent may not be logged in recovery.
TEZ-2031. Tez UI: horizontal scrollbars do not appear in tables, causing them to look truncated.
TEZ-2073. SimpleHistoryLoggingService cannot be read by log aggregation (umask)
TEZ-2078. Tez UI: Task logs url use in-progress url causing various errors.
http://git-wip-us.apache.org/repos/asf/tez/blob/b74bab47/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index aba20cf..3f9e2cf 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -512,6 +512,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
this.attempts = new LinkedHashMap<TezTaskAttemptID, TaskAttempt>();
}
recoveredState = TaskState.SCHEDULED;
+ historyTaskStartGenerated = true;
taskAttemptStatus.clear();
return recoveredState;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b74bab47/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
index 9d0e121..b8b09d0 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerContext;
import org.apache.tez.dag.app.TaskAttemptListener;
@@ -56,6 +57,7 @@ import org.apache.tez.dag.history.HistoryEventHandler;
import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.history.events.TaskAttemptFinishedEvent;
import org.apache.tez.dag.history.events.TaskAttemptStartedEvent;
+import org.apache.tez.dag.history.events.TaskFinishedEvent;
import org.apache.tez.dag.records.TaskAttemptTerminationCause;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
@@ -112,6 +114,19 @@ public class TestTaskAttemptRecovery {
}
assertEquals(expectedTimes, actualTimes);
}
+
+ void verifyTaskFinishedEvent(TezTaskID taskId, TaskState finalState, int expectedTimes) {
+ int actualTimes = 0;
+ for (DAGHistoryEvent event : events) {
+ if (event.getHistoryEvent().getEventType() == HistoryEventType.TASK_FINISHED) {
+ TaskFinishedEvent tfEvent = (TaskFinishedEvent)event.getHistoryEvent();
+ if (tfEvent.getTaskID().equals(taskId) && tfEvent.getState().equals(finalState)) {
+ actualTimes ++;
+ }
+ }
+ }
+ assertEquals(expectedTimes, actualTimes);
+ }
}
@Before
http://git-wip-us.apache.org/repos/asf/tez/blob/b74bab47/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
index f6e0162..c2185d8 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
@@ -58,6 +58,7 @@ import org.apache.tez.dag.app.dag.event.TaskEvent;
import org.apache.tez.dag.app.dag.event.TaskEventRecoverTask;
import org.apache.tez.dag.app.dag.event.TaskEventType;
import org.apache.tez.dag.app.dag.event.VertexEventType;
+import org.apache.tez.dag.app.dag.impl.TestTaskAttemptRecovery.MockHistoryEventHandler;
import org.apache.tez.dag.history.events.TaskAttemptFinishedEvent;
import org.apache.tez.dag.history.events.TaskAttemptStartedEvent;
import org.apache.tez.dag.history.events.TaskFinishedEvent;
@@ -82,6 +83,7 @@ public class TestTaskRecovery {
private Configuration conf = new Configuration();
private AppContext mockAppContext;
+ private MockHistoryEventHandler mockHistoryEventHandler;
private ApplicationId appId = ApplicationId.newInstance(
System.currentTimeMillis(), 1);
private TezDAGID dagId = TezDAGID.getInstance(appId, 1);
@@ -181,7 +183,8 @@ public class TestTaskRecovery {
mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
when(mockAppContext.getCurrentDAG().getVertex(any(TezVertexID.class)))
.thenReturn(vertex);
-
+ mockHistoryEventHandler = new MockHistoryEventHandler(mockAppContext);
+ when(mockAppContext.getHistoryHandler()).thenReturn(mockHistoryEventHandler);
task =
new TaskImpl(vertexId, 0, dispatcher.getEventHandler(),
new Configuration(), mock(TaskAttemptListener.class),
@@ -313,6 +316,7 @@ public class TestTaskRecovery {
assertEquals(0, task.failedAttempts);
assertEquals(0, task.getUncompletedAttemptsCount());
assertEquals(taId, task.successfulAttempt);
+ mockHistoryEventHandler.verifyTaskFinishedEvent(task.getTaskId(), TaskState.SUCCEEDED, 1);
}
/**
@@ -420,6 +424,7 @@ public class TestTaskRecovery {
assertEquals(0, task.failedAttempts);
assertEquals(0, task.getUncompletedAttemptsCount());
assertEquals(taId, task.successfulAttempt);
+ mockHistoryEventHandler.verifyTaskFinishedEvent(task.getTaskId(), TaskState.SUCCEEDED, 1);
}
/**
[06/23] tez git commit: TEZ-2092. Tez UI history url handler injects
spurious trailing slash. (hitesh)
Posted by ss...@apache.org.
TEZ-2092. Tez UI history url handler injects spurious trailing slash. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/6f0825d0
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/6f0825d0
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/6f0825d0
Branch: refs/heads/TEZ-2003
Commit: 6f0825d0fbee49f652e3341b82e63d13d33d59c4
Parents: 8e8405b
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Feb 13 09:30:49 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Feb 13 09:30:49 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java | 2 +-
.../apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java | 7 +++++++
3 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/6f0825d0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index df584a9..ec631b3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -62,6 +62,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2092. Tez UI history url handler injects spurious trailing slash.
TEZ-2098. Tez UI: Dag details should be the default page for dag, fix invalid time entries for failed Vertices.
TEZ-2024. TaskFinishedEvent may not be logged in recovery.
TEZ-2031. Tez UI: horizontal scrollbars do not appear in tables, causing them to look truncated.
http://git-wip-us.apache.org/repos/asf/tez/blob/6f0825d0/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 4fd655e..05cbc66 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -604,7 +604,7 @@ public class TaskSchedulerEventHandler extends AbstractService
// instances of consecutive "/" with single (except for the http(s):// case
historyUrl = historyUrlTemplate
.replaceAll(APPLICATION_ID_PLACEHOLDER, appContext.getApplicationID().toString())
- .replaceAll(HISTORY_URL_BASE, historyUrlBase + "/")
+ .replaceAll(HISTORY_URL_BASE, historyUrlBase)
.replaceAll("([^:])/{2,}", "$1/");
}
http://git-wip-us.apache.org/repos/asf/tez/blob/6f0825d0/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
index 1611859..7bb9d9b 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
@@ -280,6 +280,13 @@ public class TestTaskSchedulerEventHandler {
"__HISTORY_URL_BASE__#/somepath");
Assert.assertTrue("http://ui-host:9998/#/somepath"
.equals(schedulerHandler.getHistoryUrl()));
+
+ conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
+ "__HISTORY_URL_BASE__?viewPath=tez-app/__APPLICATION_ID__");
+ conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://localhost/ui/tez");
+ Assert.assertTrue("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID"
+ .equals(schedulerHandler.getHistoryUrl()));
+
}
}
[08/23] tez git commit: TEZ-2093. Add events to MockDAGAppMaster and
add e2e test for event routing (bikas)
Posted by ss...@apache.org.
TEZ-2093. Add events to MockDAGAppMaster and add e2e test for event routing (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/48fd4d6b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/48fd4d6b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/48fd4d6b
Branch: refs/heads/TEZ-2003
Commit: 48fd4d6b5bca8782821e70ba0689ad43ca27e442
Parents: ada11cc
Author: Bikas Saha <bi...@apache.org>
Authored: Fri Feb 13 11:20:05 2015 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri Feb 13 11:20:05 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../dag/app/TaskAttemptListenerImpTezDag.java | 3 -
.../apache/tez/dag/app/dag/impl/TaskImpl.java | 7 ++
.../apache/tez/dag/app/MockDAGAppMaster.java | 26 ++++++-
.../tez/dag/app/TestMockDAGAppMaster.java | 82 ++++++++++++++++++++
5 files changed, 115 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/48fd4d6b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9f33a85..d316290 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.7.0: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2093. Add events to MockDAGAppMaster and add e2e test for event routing
TEZ-2075. Incompatible issue caused by TEZ-1233 that TezConfiguration.TEZ_SITE_XML is made private
TEZ-2082. Race condition in TaskAttemptListenerImpTezDag.getTask()
TEZ-1233. Allow configuration of framework parameters per vertex.
http://git-wip-us.apache.org/repos/asf/tez/blob/48fd4d6b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index d8e24c0..4cb5e99 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -62,7 +62,6 @@ import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
import org.apache.tez.common.security.JobTokenSecretManager;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
@SuppressWarnings("unchecked")
@@ -223,8 +222,6 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
LOG.info("Container with id: " + containerId
+ " is valid, but no longer registered, and will be killed. Race condition.");
} else {
- Preconditions.checkState(task != null && task != TASK_FOR_INVALID_JVM, "CId: "
- + containerId);
context.getEventHandler().handle(
new TaskAttemptEventStartedRemotely(task.getTaskSpec()
.getTaskAttemptID(), containerId, context
http://git-wip-us.apache.org/repos/asf/tez/blob/48fd4d6b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 3f9e2cf..125eeed 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -37,6 +37,7 @@ import com.google.common.collect.Maps;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -1443,6 +1444,12 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
this.writeLock.unlock();
}
}
+
+ @Private
+ @VisibleForTesting
+ public List<TezEvent> getTaskEvents() {
+ return tezEventsForTaskAttempts;
+ }
private static class KillTransition
implements SingleArcTransition<TaskImpl, TaskEvent> {
http://git-wip-us.apache.org/repos/asf/tez/blob/48fd4d6b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index d34532b..04a47c6 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -31,7 +31,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
@@ -54,9 +53,14 @@ import org.apache.tez.dag.app.rm.container.AMContainerEventType;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.events.CompositeDataMovementEvent;
+import org.apache.tez.runtime.api.events.DataMovementEvent;
import org.apache.tez.runtime.api.events.TaskAttemptCompletedEvent;
import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent;
import org.apache.tez.runtime.api.impl.EventMetaData;
+import org.apache.tez.runtime.api.impl.OutputSpec;
+import org.apache.tez.runtime.api.impl.TaskSpec;
import org.apache.tez.runtime.api.impl.TezEvent;
import org.apache.tez.runtime.api.impl.EventMetaData.EventProducerConsumerType;
@@ -69,6 +73,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
MockContainerLauncher containerLauncher;
boolean initFailFlag;
boolean startFailFlag;
+ boolean sendDMEvents;
// mock container launcher does not launch real tasks.
// Upon, launch of a container is simulates the container asking for tasks
@@ -99,6 +104,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
ContainerId cId;
TezTaskAttemptID taId;
String vName;
+ TaskSpec taskSpec;
ContainerLaunchContext launchContext;
int numUpdates = 0;
boolean completed;
@@ -111,6 +117,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
void clear() {
taId = null;
vName = null;
+ taskSpec = null;
completed = false;
launchContext = null;
}
@@ -215,7 +222,9 @@ public class MockDAGAppMaster extends DAGAppMaster {
@Override
public void run() {
// wait for test to sync with us and get a reference to us. Go when sync is done
+ LOG.info("Waiting to go");
waitToGo();
+ LOG.info("Signal to go");
while(true) {
if (!startScheduling.get()) { // schedule when asked to do so by the test code
continue;
@@ -236,6 +245,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
} else {
cData.taId = cTask.getTaskSpec().getTaskAttemptID();
cData.vName = cTask.getTaskSpec().getVertexName();
+ cData.taskSpec = cTask.getTaskSpec();
}
} catch (IOException e) {
e.printStackTrace();
@@ -261,6 +271,20 @@ public class MockDAGAppMaster extends DAGAppMaster {
// send a done notification
TezVertexID vertexId = cData.taId.getTaskID().getVertexID();
cData.completed = true;
+ if (sendDMEvents) {
+ Event event = null;
+ for (OutputSpec output : cData.taskSpec.getOutputs()) {
+ if (output.getPhysicalEdgeCount() == 1) {
+ event = DataMovementEvent.create(0, null);
+ } else {
+ event = CompositeDataMovementEvent.create(0, output.getPhysicalEdgeCount(), null);
+ }
+ getContext().getEventHandler().handle(
+ new VertexEventRouteEvent(vertexId, Collections.singletonList(new TezEvent(
+ event, new EventMetaData(EventProducerConsumerType.OUTPUT, cData.vName,
+ output.getDestinationVertexName(), cData.taId)))));
+ }
+ }
getContext().getEventHandler().handle(
new VertexEventRouteEvent(vertexId, Collections.singletonList(new TezEvent(
new TaskAttemptCompletedEvent(), new EventMetaData(
http://git-wip-us.apache.org/repos/asf/tez/blob/48fd4d6b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
index a46821a..f4734d5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
@@ -19,6 +19,7 @@
package org.apache.tez.dag.app;
import java.io.IOException;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -31,9 +32,16 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
+import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
+import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
@@ -43,9 +51,13 @@ import org.apache.tez.dag.app.dag.DAGState;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.app.dag.event.DAGAppMasterEventSchedulingServiceError;
import org.apache.tez.dag.app.dag.impl.DAGImpl;
+import org.apache.tez.dag.app.dag.impl.TaskImpl;
+import org.apache.tez.dag.app.dag.impl.VertexImpl;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.api.events.DataMovementEvent;
+import org.apache.tez.runtime.api.impl.TezEvent;
import org.junit.Assert;
import org.junit.Test;
@@ -140,6 +152,76 @@ public class TestMockDAGAppMaster {
tezClient.stop();
}
+ @Test (timeout = 5000)
+ public void testBasicEvents() throws Exception {
+ TezConfiguration tezconf = new TezConfiguration(defaultConf);
+
+ MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
+ tezClient.start();
+
+ MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
+ MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
+ mockLauncher.startScheduling(false);
+ mockApp.sendDMEvents = true;
+ DAG dag = DAG.create("test");
+ Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 2);
+ Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 2);
+ Vertex vC = Vertex.create("C", ProcessorDescriptor.create("Proc.class"), 2);
+ Vertex vD = Vertex.create("D", ProcessorDescriptor.create("Proc.class"), 2);
+ dag.addVertex(vA)
+ .addVertex(vB)
+ .addVertex(vC)
+ .addVertex(vD)
+ .addEdge(
+ Edge.create(vA, vB, EdgeProperty.create(DataMovementType.BROADCAST,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("Out"), InputDescriptor.create("In"))))
+ .addEdge(
+ Edge.create(vA, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("Out"), InputDescriptor.create("In"))))
+ .addEdge(
+ Edge.create(vA, vD, EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
+
+ DAGClient dagClient = tezClient.submitDAG(dag);
+ mockLauncher.waitTillContainersLaunched();
+ DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
+ mockLauncher.startScheduling(true);
+ dagClient.waitForCompletion();
+ Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
+ VertexImpl vImpl = (VertexImpl) dagImpl.getVertex(vB.getName());
+ TaskImpl tImpl = (TaskImpl) vImpl.getTask(1);
+ List<TezEvent> tEvents = tImpl.getTaskEvents();
+ Assert.assertEquals(2, tEvents.size()); // 2 from vA
+ Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
+ Assert.assertEquals(0, ((DataMovementEvent)tEvents.get(0).getEvent()).getTargetIndex());
+ Assert.assertEquals(0, ((DataMovementEvent)tEvents.get(0).getEvent()).getSourceIndex());
+ Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
+ Assert.assertEquals(1, ((DataMovementEvent)tEvents.get(1).getEvent()).getTargetIndex());
+ Assert.assertEquals(0, ((DataMovementEvent)tEvents.get(1).getEvent()).getSourceIndex());
+ vImpl = (VertexImpl) dagImpl.getVertex(vC.getName());
+ tImpl = (TaskImpl) vImpl.getTask(1);
+ tEvents = tImpl.getTaskEvents();
+ Assert.assertEquals(2, tEvents.size()); // 2 from vA
+ Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
+ Assert.assertEquals(0, ((DataMovementEvent)tEvents.get(0).getEvent()).getTargetIndex());
+ Assert.assertEquals(1, ((DataMovementEvent)tEvents.get(0).getEvent()).getSourceIndex());
+ Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
+ Assert.assertEquals(1, ((DataMovementEvent)tEvents.get(1).getEvent()).getTargetIndex());
+ Assert.assertEquals(1, ((DataMovementEvent)tEvents.get(1).getEvent()).getSourceIndex());
+ vImpl = (VertexImpl) dagImpl.getVertex(vD.getName());
+ tImpl = (TaskImpl) vImpl.getTask(1);
+ tEvents = tImpl.getTaskEvents();
+ Assert.assertEquals(1, tEvents.size()); // 1 from vA
+ Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
+ Assert.assertEquals(0, ((DataMovementEvent)tEvents.get(0).getEvent()).getTargetIndex());
+ Assert.assertEquals(0, ((DataMovementEvent)tEvents.get(0).getEvent()).getSourceIndex());
+
+ tezClient.stop();
+ }
+
@Test (timeout = 10000)
public void testMultipleSubmissions() throws Exception {
Map<String, LocalResource> lrDAG = Maps.newHashMap();