You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2015/06/18 20:13:24 UTC
tez git commit: TEZ-2549. Reduce Counter Load on the Timeline Server
(jlowe via jeagles)
Repository: tez
Updated Branches:
refs/heads/master 5cba084eb -> 43fcf38bc
TEZ-2549. Reduce Counter Load on the Timeline Server (jlowe via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/43fcf38b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/43fcf38b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/43fcf38b
Branch: refs/heads/master
Commit: 43fcf38bc73473cba0b775232d36383b3d332fd8
Parents: 5cba084
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Thu Jun 18 13:13:08 2015 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Thu Jun 18 13:13:08 2015 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../dag/api/client/DAGClientTimelineImpl.java | 5 +++--
.../apache/tez/dag/history/utils/DAGUtils.java | 22 +++++++++++++-------
.../apache/tez/history/parser/utils/Utils.java | 4 ++--
.../app/scripts/components/counter-table.js | 4 ++--
.../app/templates/components/counter-table.hbs | 15 +++++++++++--
6 files changed, 36 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/43fcf38b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 36e17ec..a0c0c7e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -255,6 +255,7 @@ Release 0.6.2: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2549. Reduce Counter Load on the Timeline Server
TEZ-2548. TezClient submitDAG can hang if the AM is in the process of shutting down.
TEZ-2534. Error handling summary event when shutting down AM.
TEZ-2511. Add exitCode to diagnostics when container fails.
http://git-wip-us.apache.org/repos/asf/tez/blob/43fcf38b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
index 9a0949b..2294759 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
@@ -348,7 +348,7 @@ public class DAGClientTimelineImpl extends DAGClient {
final String groupName = counterGroupNode.optString(ATSConstants.COUNTER_GROUP_NAME);
final String groupDisplayName = counterGroupNode.optString(
- ATSConstants.COUNTER_GROUP_DISPLAY_NAME);
+ ATSConstants.COUNTER_GROUP_DISPLAY_NAME, groupName);
final JSONArray counterNodes = counterGroupNode.optJSONArray(ATSConstants.COUNTERS);
final int numCounters = counterNodes.length();
@@ -357,7 +357,8 @@ public class DAGClientTimelineImpl extends DAGClient {
for (int i = 0; i < numCounters; i++) {
final JSONObject counterNode = counterNodes.getJSONObject(i);
final String counterName = counterNode.getString(ATSConstants.COUNTER_NAME);
- final String counterDisplayName = counterNode.getString(ATSConstants.COUNTER_DISPLAY_NAME);
+ final String counterDisplayName = counterNode.optString(ATSConstants.COUNTER_DISPLAY_NAME,
+ counterName);
final long counterValue = counterNode.getLong(ATSConstants.COUNTER_VALUE);
counters.add(
http://git-wip-us.apache.org/repos/asf/tez/blob/43fcf38b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index 3ec9900..1447832 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -113,21 +113,29 @@ public class DAGUtils {
}
ArrayList<Object> counterGroupsList = new ArrayList<Object>();
for (CounterGroup group : counters) {
- Map<String,Object> counterGroupMap = new LinkedHashMap<String, Object>();
- counterGroupMap.put(ATSConstants.COUNTER_GROUP_NAME, group.getName());
- counterGroupMap.put(ATSConstants.COUNTER_GROUP_DISPLAY_NAME,
- group.getDisplayName());
ArrayList<Object> counterList = new ArrayList<Object>();
for (TezCounter counter : group) {
+ if (counter.getValue() != 0) {
Map<String,Object> counterMap = new LinkedHashMap<String, Object>();
counterMap.put(ATSConstants.COUNTER_NAME, counter.getName());
- counterMap.put(ATSConstants.COUNTER_DISPLAY_NAME,
+ if (!counter.getDisplayName().equals(counter.getName())) {
+ counterMap.put(ATSConstants.COUNTER_DISPLAY_NAME,
counter.getDisplayName());
+ }
counterMap.put(ATSConstants.COUNTER_VALUE, counter.getValue());
counterList.add(counterMap);
}
- putInto(counterGroupMap, ATSConstants.COUNTERS, counterList);
- counterGroupsList.add(counterGroupMap);
+ }
+ if (!counterList.isEmpty()) {
+ Map<String,Object> counterGroupMap = new LinkedHashMap<String, Object>();
+ counterGroupMap.put(ATSConstants.COUNTER_GROUP_NAME, group.getName());
+ if (!group.getDisplayName().equals(group.getName())) {
+ counterGroupMap.put(ATSConstants.COUNTER_GROUP_DISPLAY_NAME,
+ group.getDisplayName());
+ }
+ counterGroupMap.put(ATSConstants.COUNTERS, counterList);
+ counterGroupsList.add(counterGroupMap);
+ }
}
putInto(object, ATSConstants.COUNTER_GROUPS, counterGroupsList);
return object;
http://git-wip-us.apache.org/repos/asf/tez/blob/43fcf38b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java
index d5a66e3..7345012 100644
--- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java
+++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java
@@ -61,7 +61,7 @@ public class Utils {
JSONObject counterGroupNode = counterGroupNodes.optJSONObject(i);
final String groupName = counterGroupNode.optString(Constants.COUNTER_GROUP_NAME);
final String groupDisplayName = counterGroupNode.optString(
- Constants.COUNTER_GROUP_DISPLAY_NAME);
+ Constants.COUNTER_GROUP_DISPLAY_NAME, groupName);
CounterGroup group = counters.addGroup(groupName, groupDisplayName);
@@ -72,7 +72,7 @@ public class Utils {
JSONObject counterNode = counterNodes.optJSONObject(j);
final String counterName = counterNode.getString(Constants.COUNTER_NAME);
final String counterDisplayName =
- counterNode.getString(Constants.COUNTER_DISPLAY_NAME);
+ counterNode.optString(Constants.COUNTER_DISPLAY_NAME, counterName);
final long counterValue = counterNode.getLong(Constants.COUNTER_VALUE);
TezCounter counter = group.findCounter(
counterName,
http://git-wip-us.apache.org/repos/asf/tez/blob/43fcf38b/tez-ui/src/main/webapp/app/scripts/components/counter-table.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/components/counter-table.js b/tez-ui/src/main/webapp/app/scripts/components/counter-table.js
index 8d6f37d..260f68d 100644
--- a/tez-ui/src/main/webapp/app/scripts/components/counter-table.js
+++ b/tez-ui/src/main/webapp/app/scripts/components/counter-table.js
@@ -41,13 +41,13 @@ App.CounterTableComponent = Em.Component.extend({
rawData.forEach(function(cg) {
var tmpcg = {
counterGroupName: cg['counterGroupName'],
- counterGroupDisplayName: cg['counterGroupDisplayName'],
+ counterGroupDisplayName: cg['counterGroupDisplayName'] || cg['counterGroupName'],
counters: []
};
var counters = cg['counters'] || [];
counters.forEach(function(counter) {
- if (filterStringRegex.test(counter['counterDisplayName'])) {
+ if (filterStringRegex.test(counter['counterDisplayName'] || counter['counterName'])) {
tmpcg.counters.push(counter);
}
});
http://git-wip-us.apache.org/repos/asf/tez/blob/43fcf38b/tez-ui/src/main/webapp/app/templates/components/counter-table.hbs
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/templates/components/counter-table.hbs b/tez-ui/src/main/webapp/app/templates/components/counter-table.hbs
index 4e8f603..fcbbd15 100644
--- a/tez-ui/src/main/webapp/app/templates/components/counter-table.hbs
+++ b/tez-ui/src/main/webapp/app/templates/components/counter-table.hbs
@@ -33,11 +33,22 @@
<tbody>
{{#each counterGroup in filteredData}}
<tr class='group-header'>
- <td colspan='2'>{{counterGroup.counterGroupDisplayName}}</td>
+ <td colspan='2'>
+ {{#if counterGroup.counterGroupDisplayName}}
+ {{counterGroup.counterGroupDisplayName}}
+ {{else}}
+ {{counterGroup.counterGroupName}}
+ {{/if}}
+ </td>
</tr>
{{#each counter in counterGroup.counters}}
<tr>
- <td>{{counter.counterDisplayName}}</td>
+ <td>{{#if counter.counterDisplayName}}
+ {{counter.counterDisplayName}}
+ {{else}}
+ {{counter.counterName}}
+ {{/if}}
+ </td>
<td>{{formatNumThousands counter.counterValue}}</td>
</tr>
{{/each}}