You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2015/03/27 01:57:26 UTC
[08/52] [abbrv] tez git commit: TEZ-2179. Timeline relatedentries
missing cause exaggerated warning. (Chang Li via hitesh)
TEZ-2179. Timeline relatedentries missing cause exaggerated warning. (Chang Li via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/19df2778
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/19df2778
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/19df2778
Branch: refs/heads/TEZ-2003
Commit: 19df277868e2775bb92ac80c6c7fd147724f9268
Parents: e0e1912
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Mar 12 07:46:43 2015 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Mar 12 07:47:47 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../ats/HistoryEventTimelineConversion.java | 32 +++++-------
.../ats/TestHistoryEventTimelineConversion.java | 53 ++++++++------------
3 files changed, 35 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/19df2778/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 44a13b7..d29cb5d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -80,6 +80,7 @@ Release 0.6.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2179. Timeline relatedentries missing cause exaggerated warning.
TEZ-2168. Fix application dependencies on mutually exclusive artifacts: tez-yarn-timeline-history
and tez-yarn-timeline-history-with-acls.
TEZ-2190. TestOrderedWordCount fails when generateSplitsInClient set to true.
http://git-wip-us.apache.org/repos/asf/tez/blob/19df2778/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
index ca47b92..e1c2a72 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
@@ -135,14 +135,13 @@ public class HistoryEventTimelineConversion {
+ event.getApplicationId().toString());
atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION.name());
- atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID,
- event.getApplicationId().toString());
- atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser());
-
atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
atsEntity.addOtherInfo(ATSConstants.CONFIG,
DAGUtils.convertConfigurationToATSMap(event.getConf()));
+ atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID,
+ event.getApplicationId().toString());
+ atsEntity.addOtherInfo(ATSConstants.USER, event.getUser());
atsEntity.setStartTime(event.getLaunchTime());
@@ -160,12 +159,6 @@ public class HistoryEventTimelineConversion {
+ event.getApplicationAttemptId().toString());
atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
- atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID,
- event.getApplicationAttemptId().getApplicationId().toString());
- atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ATTEMPT_ID,
- event.getApplicationAttemptId().toString());
- atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser());
-
atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
event.getApplicationAttemptId().getApplicationId().toString());
@@ -178,6 +171,11 @@ public class HistoryEventTimelineConversion {
atsEntity.addEvent(launchEvt);
atsEntity.addOtherInfo(ATSConstants.APP_SUBMIT_TIME, event.getAppSubmitTime());
+ atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID,
+ event.getApplicationAttemptId().getApplicationId().toString());
+ atsEntity.addOtherInfo(ATSConstants.APPLICATION_ATTEMPT_ID,
+ event.getApplicationAttemptId().toString());
+ atsEntity.addOtherInfo(ATSConstants.USER, event.getUser());
return atsEntity;
}
@@ -208,12 +206,12 @@ public class HistoryEventTimelineConversion {
atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
"tez_" + event.getApplicationAttemptId().toString());
- atsEntity.addRelatedEntity(ATSConstants.CONTAINER_ID,
- event.getContainerId().toString());
atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
event.getApplicationAttemptId().getApplicationId().toString());
+ atsEntity.addOtherInfo(ATSConstants.CONTAINER_ID,
+ event.getContainerId().toString());
atsEntity.setStartTime(event.getLaunchTime());
TimelineEvent launchEvt = new TimelineEvent();
@@ -341,11 +339,6 @@ public class HistoryEventTimelineConversion {
"tez_" + event.getApplicationAttemptId().getApplicationId().toString());
atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
"tez_" + event.getApplicationAttemptId().toString());
- atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID,
- event.getApplicationAttemptId().getApplicationId().toString());
- atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ATTEMPT_ID,
- event.getApplicationAttemptId().toString());
- atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser());
TimelineEvent submitEvt = new TimelineEvent();
submitEvt.setEventType(HistoryEventType.DAG_SUBMITTED.name());
@@ -367,6 +360,9 @@ public class HistoryEventTimelineConversion {
}
atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID,
event.getApplicationAttemptId().getApplicationId().toString());
+ atsEntity.addOtherInfo(ATSConstants.APPLICATION_ATTEMPT_ID,
+ event.getApplicationAttemptId().toString());
+ atsEntity.addOtherInfo(ATSConstants.USER, event.getUser());
return atsEntity;
}
@@ -412,8 +408,6 @@ public class HistoryEventTimelineConversion {
atsEntity.setStartTime(event.getStartTime());
- atsEntity.addRelatedEntity(ATSConstants.NODE_ID, event.getNodeId().toString());
- atsEntity.addRelatedEntity(ATSConstants.CONTAINER_ID, event.getContainerId().toString());
atsEntity.addRelatedEntity(EntityTypes.TEZ_TASK_ID.name(),
event.getTaskAttemptID().getTaskID().toString());
http://git-wip-us.apache.org/repos/asf/tez/blob/19df2778/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index 6d713c5..a2b0f89 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -222,18 +222,17 @@ public class TestHistoryEventTimelineConversion {
Assert.assertEquals(EntityTypes.TEZ_APPLICATION.name(), timelineEntity.getEntityType());
Assert.assertEquals("tez_" + applicationId.toString(), timelineEntity.getEntityId());
- Assert.assertEquals(2, timelineEntity.getRelatedEntities().size());
- Assert.assertTrue(timelineEntity.getRelatedEntities().get(ATSConstants.USER).contains(user));
- Assert.assertTrue(
- timelineEntity.getRelatedEntities().get(ATSConstants.APPLICATION_ID).contains(
- applicationId.toString()));
+ Assert.assertEquals(0, timelineEntity.getRelatedEntities().size());
Assert.assertEquals(1, timelineEntity.getPrimaryFilters().size());
Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.USER).contains(user));
- Assert.assertEquals(2, timelineEntity.getOtherInfo().size());
+ Assert.assertEquals(4, timelineEntity.getOtherInfo().size());
Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.CONFIG));
Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.TEZ_VERSION));
+ Assert.assertEquals(user, timelineEntity.getOtherInfo().get(ATSConstants.USER));
+ Assert.assertEquals(applicationId.toString(),
+ timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ID));
Map<String, String> config =
(Map<String, String>)timelineEntity.getOtherInfo().get(ATSConstants.CONFIG);
@@ -263,12 +262,7 @@ public class TestHistoryEventTimelineConversion {
Assert.assertEquals(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), timelineEntity.getEntityType());
final Map<String, Set<String>> relatedEntities = timelineEntity.getRelatedEntities();
- Assert.assertEquals(3, relatedEntities.size());
- Assert.assertTrue(relatedEntities.get(ATSConstants.APPLICATION_ID)
- .contains(applicationId.toString()));
- Assert.assertTrue(relatedEntities.get(ATSConstants.APPLICATION_ATTEMPT_ID)
- .contains(applicationAttemptId.toString()));
- Assert.assertTrue(relatedEntities.get(ATSConstants.USER).contains(user));
+ Assert.assertEquals(0, relatedEntities.size());
final Map<String, Set<Object>> primaryFilters = timelineEntity.getPrimaryFilters();
Assert.assertEquals(2, primaryFilters.size());
@@ -284,8 +278,11 @@ public class TestHistoryEventTimelineConversion {
Assert.assertEquals(launchTime, evt.getTimestamp());
final Map<String, Object> otherInfo = timelineEntity.getOtherInfo();
- Assert.assertEquals(1, otherInfo.size());
+ Assert.assertEquals(4, otherInfo.size());
Assert.assertEquals(submitTime, otherInfo.get(ATSConstants.APP_SUBMIT_TIME));
+ Assert.assertEquals(applicationId.toString(), otherInfo.get(ATSConstants.APPLICATION_ID));
+ Assert.assertEquals(applicationAttemptId.toString(), otherInfo.get(ATSConstants.APPLICATION_ATTEMPT_ID));
+ Assert.assertEquals(user, otherInfo.get(ATSConstants.USER));
}
@Test(timeout = 5000)
@@ -298,9 +295,7 @@ public class TestHistoryEventTimelineConversion {
Assert.assertEquals(EntityTypes.TEZ_CONTAINER_ID.name(), timelineEntity.getEntityType());
Assert.assertEquals("tez_" + containerId.toString(), timelineEntity.getEntityId());
- Assert.assertEquals(2, timelineEntity.getRelatedEntities().size());
- Assert.assertTrue(timelineEntity.getRelatedEntities().get(ATSConstants.CONTAINER_ID).contains(
- containerId.toString()));
+ Assert.assertEquals(1, timelineEntity.getRelatedEntities().size());
Assert.assertTrue(
timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION_ATTEMPT.name()).contains(
"tez_" + applicationAttemptId.toString()));
@@ -309,6 +304,8 @@ public class TestHistoryEventTimelineConversion {
Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.APPLICATION_ID).contains(
applicationAttemptId.getApplicationId().toString()));
+ Assert.assertEquals(containerId.toString(),timelineEntity.getOtherInfo().get(ATSConstants.CONTAINER_ID));
+
Assert.assertEquals(launchTime, timelineEntity.getStartTime().longValue());
Assert.assertEquals(1, timelineEntity.getEvents().size());
@@ -389,21 +386,13 @@ public class TestHistoryEventTimelineConversion {
Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType());
Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId());
- Assert.assertEquals(5, timelineEntity.getRelatedEntities().size());
+ Assert.assertEquals(2, timelineEntity.getRelatedEntities().size());
Assert.assertTrue(
timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION.name()).contains(
"tez_" + applicationId.toString()));
Assert.assertTrue(
timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION_ATTEMPT.name()).contains(
"tez_" + applicationAttemptId.toString()));
- Assert.assertTrue(
- timelineEntity.getRelatedEntities().get(ATSConstants.APPLICATION_ATTEMPT_ID).contains(
- applicationAttemptId.toString()));
- Assert.assertTrue(
- timelineEntity.getRelatedEntities().get(ATSConstants.APPLICATION_ID).contains(
- applicationAttemptId.getApplicationId().toString()));
- Assert.assertTrue(
- timelineEntity.getRelatedEntities().get(ATSConstants.USER).contains(user));
Assert.assertEquals(1, timelineEntity.getEvents().size());
TimelineEvent timelineEvent = timelineEntity.getEvents().get(0);
@@ -426,7 +415,12 @@ public class TestHistoryEventTimelineConversion {
Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.DAG_PLAN));
Assert.assertEquals(applicationId.toString(),
timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ID));
-
+ Assert.assertEquals(applicationAttemptId.toString(),
+ timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ATTEMPT_ID));
+ Assert.assertEquals(applicationAttemptId.getApplicationId().toString(),
+ timelineEntity.getOtherInfo().get(ATSConstants.APPLICATION_ID));
+ Assert.assertEquals(user,
+ timelineEntity.getOtherInfo().get(ATSConstants.USER));
}
@Test(timeout = 5000)
@@ -721,12 +715,7 @@ public class TestHistoryEventTimelineConversion {
Assert.assertEquals(startTime, timelineEntity.getStartTime().longValue());
- Assert.assertEquals(3, timelineEntity.getRelatedEntities().size());
- Assert.assertTrue(
- timelineEntity.getRelatedEntities().get(ATSConstants.NODE_ID).contains(nodeId.toString()));
- Assert.assertTrue(
- timelineEntity.getRelatedEntities().get(ATSConstants.CONTAINER_ID).contains(
- containerId.toString()));
+ Assert.assertEquals(1, timelineEntity.getRelatedEntities().size());
Assert.assertTrue(
timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_TASK_ID.name()).contains(
tezTaskID.toString()));