You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by zh...@apache.org on 2017/03/28 19:47:46 UTC
[05/50] [abbrv] tez git commit: TEZ-3609. Improve ATSv15 performance
for DAG entities read calls. Contributed by Harish Jaiprakash.
TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
Contributed by Harish Jaiprakash.
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/11815a7c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/11815a7c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/11815a7c
Branch: refs/heads/TEZ-1190
Commit: 11815a7ccd80ac3cb052f34a420d44de9f083542
Parents: 63ae97d
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Feb 7 22:54:08 2017 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Feb 7 22:54:08 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../logging/ats/TimelineCachePluginImpl.java | 1 -
.../ats/TestTimelineCachePluginImpl.java | 41 ++++++++++----------
3 files changed, 23 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/11815a7c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a7cc0ce..0949339 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
TEZ-3244. Allow overlap of input and output memory when they are not concurrent
TEZ-3581. Add different logger to enable suppressing logs for specific lines.
TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels
@@ -198,6 +199,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
TEZ-3244. Allow overlap of input and output memory when they are not concurrent
TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels
TEZ-3462. Task attempt failure during container shutdown loses useful container diagnostics
http://git-wip-us.apache.org/repos/asf/tez/blob/11815a7c/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java
index b4217a1..8269714 100644
--- a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java
+++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java
@@ -72,7 +72,6 @@ public class TimelineCachePluginImpl extends TimelineEntityGroupPlugin implement
private Set<TimelineEntityGroupId> createTimelineEntityGroupIds(TezDAGID dagId) {
ApplicationId appId = dagId.getApplicationId();
HashSet<TimelineEntityGroupId> groupIds = Sets.newHashSet(
- TimelineEntityGroupId.newInstance(appId, appId.toString()),
TimelineEntityGroupId.newInstance(appId, dagId.toString()));
for (int numGroupsPerDag : allNumGroupsPerDag) {
groupIds.add(TimelineEntityGroupId.newInstance(appId, dagId.getGroupId(numGroupsPerDag)));
http://git-wip-us.apache.org/repos/asf/tez/blob/11815a7c/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java
index 6f819ba..3d1af63 100644
--- a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java
+++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java
@@ -124,12 +124,12 @@ public class TestTimelineCachePluginImpl {
Assert.assertNull(groupIds);
continue;
}
- Assert.assertEquals(3, groupIds.size());
+ Assert.assertEquals(2, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId1, groupId.getApplicationId());
- Assert.assertTrue(getGroupIds(dagID1, appId1, 100).contains(groupId.getTimelineEntityGroupId()));
+ Assert.assertTrue(getGroupIds(dagID1, 100).contains(groupId.getTimelineEntityGroupId()));
}
}
}
@@ -143,12 +143,12 @@ public class TestTimelineCachePluginImpl {
Assert.assertNull(groupIds);
continue;
}
- Assert.assertEquals(2, groupIds.size());
+ Assert.assertEquals(1, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId1, groupId.getApplicationId());
- Assert.assertTrue(getGroupIds(dagID1, appId1).contains(groupId.getTimelineEntityGroupId()));
+ Assert.assertTrue(getGroupIds(dagID1).contains(groupId.getTimelineEntityGroupId()));
}
}
}
@@ -162,12 +162,12 @@ public class TestTimelineCachePluginImpl {
Assert.assertNull(groupIds);
continue;
}
- Assert.assertEquals(2, groupIds.size());
+ Assert.assertEquals(1, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId1, groupId.getApplicationId());
- Assert.assertTrue(getGroupIds(dagID1, appId1).contains(groupId.getTimelineEntityGroupId()));
+ Assert.assertTrue(getGroupIds(dagID1).contains(groupId.getTimelineEntityGroupId()));
}
}
}
@@ -181,12 +181,12 @@ public class TestTimelineCachePluginImpl {
Assert.assertNull(groupIds);
continue;
}
- Assert.assertEquals(3, groupIds.size());
+ Assert.assertEquals(2, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId1, groupId.getApplicationId());
- Assert.assertTrue(getGroupIds(dagID1, appId1, 100).contains(groupId.getTimelineEntityGroupId()));
+ Assert.assertTrue(getGroupIds(dagID1, 100).contains(groupId.getTimelineEntityGroupId()));
}
}
}
@@ -200,12 +200,12 @@ public class TestTimelineCachePluginImpl {
Assert.assertNull(groupIds);
continue;
}
- Assert.assertEquals(4, groupIds.size());
+ Assert.assertEquals(3, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId2, groupId.getApplicationId());
- Assert.assertTrue(getGroupIds(dagID2, appId2, 100, 50).contains(groupId.getTimelineEntityGroupId()));
+ Assert.assertTrue(getGroupIds(dagID2, 100, 50).contains(groupId.getTimelineEntityGroupId()));
}
}
}
@@ -219,12 +219,13 @@ public class TestTimelineCachePluginImpl {
Assert.assertNull(groupIds);
continue;
}
- Assert.assertEquals(5, groupIds.size());
+ Assert.assertEquals(4, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId2, groupId.getApplicationId());
- Assert.assertTrue(getGroupIds(dagID2, appId2, 100, 25, 50).contains(groupId.getTimelineEntityGroupId()));
+ Assert.assertTrue(
+ getGroupIds(dagID2, 100, 25, 50).contains(groupId.getTimelineEntityGroupId()));
}
}
}
@@ -238,12 +239,12 @@ public class TestTimelineCachePluginImpl {
Assert.assertNull(groupIds);
continue;
}
- Assert.assertEquals(3, groupIds.size());
+ Assert.assertEquals(2, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId2, groupId.getApplicationId());
- Assert.assertTrue(getGroupIds(dagID2, appId2, 100).contains(groupId.getTimelineEntityGroupId()));
+ Assert.assertTrue(getGroupIds(dagID2, 100).contains(groupId.getTimelineEntityGroupId()));
}
}
}
@@ -261,21 +262,21 @@ public class TestTimelineCachePluginImpl {
Assert.assertNull(groupIds);
continue;
}
- Assert.assertEquals(6, groupIds.size());
+ Assert.assertEquals(4, groupIds.size());
int found = 0;
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
if (groupId.getApplicationId().equals(appId1)) {
String entityGroupId = groupId.getTimelineEntityGroupId();
- if (getGroupIds(dagID1, appId1, 100).contains(entityGroupId)) {
+ if (getGroupIds(dagID1, 100).contains(entityGroupId)) {
++found;
} else {
Assert.fail("Unexpected group id: " + entityGroupId);
}
} else if (groupId.getApplicationId().equals(appId2)) {
String entityGroupId = groupId.getTimelineEntityGroupId();
- if (getGroupIds(dagID2, appId2, 100).contains(entityGroupId)) {
+ if (getGroupIds(dagID2, 100).contains(entityGroupId)) {
++found;
} else {
Assert.fail("Unexpected group id: " + entityGroupId);
@@ -284,7 +285,7 @@ public class TestTimelineCachePluginImpl {
Assert.fail("Unexpected appId: " + groupId.getApplicationId());
}
}
- Assert.assertEquals("All groupIds not returned", 6, found);
+ Assert.assertEquals("All groupIds not returned", 4, found);
}
}
@@ -374,8 +375,8 @@ public class TestTimelineCachePluginImpl {
Assert.assertEquals("All groupIds not returned", 1, found);
}
- private Set<String> getGroupIds(TezDAGID dagId, ApplicationId appId, int ... allNumDagsPerGroup) {
- HashSet<String> groupIds = Sets.newHashSet(dagId.toString(), appId.toString());
+ private Set<String> getGroupIds(TezDAGID dagId, int ... allNumDagsPerGroup) {
+ HashSet<String> groupIds = Sets.newHashSet(dagId.toString());
for (int numDagsPerGroup : allNumDagsPerGroup) {
groupIds.add(dagId.getGroupId(numDagsPerGroup));
}