You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2015/01/19 23:39:04 UTC
tez git commit: TEZ-1931. Publish tez version info to Timeline.
(hitesh)
Repository: tez
Updated Branches:
refs/heads/branch-0.5 4bbd257db -> 6607e9448
TEZ-1931. Publish tez version info to Timeline. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/6607e944
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/6607e944
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/6607e944
Branch: refs/heads/branch-0.5
Commit: 6607e9448bae7dc24a777485dedb2e5cf787ee83
Parents: 4bbd257
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon Jan 19 14:38:36 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon Jan 19 14:38:36 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/common/ATSConstants.java | 6 +++++
.../java/org/apache/tez/common/VersionInfo.java | 14 ++++++++++
.../org/apache/tez/dag/app/DAGAppMaster.java | 3 ++-
.../dag/history/events/AppLaunchedEvent.java | 10 +++++--
.../apache/tez/dag/history/utils/DAGUtils.java | 10 +++++++
.../TestHistoryEventsProtoConversion.java | 2 +-
.../impl/TestHistoryEventJsonConversion.java | 2 +-
.../ats/HistoryEventTimelineConversion.java | 7 ++++-
.../ats/TestHistoryEventTimelineConversion.java | 28 +++++++++++++++++---
10 files changed, 73 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7244db8..81ab0e2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,7 @@ Apache Tez Change Log
Release 0.5.4: Unreleased
ALL CHANGES:
+ TEZ-1931. Publish tez version info to Timeline.
TEZ-1942. Number of tasks show in Tez UI with auto-reduce parallelism is misleading.
TEZ-1962. Fix a thread leak in LocalMode.
TEZ-1924. Tez AM does not register with AM with full FQDN causing jobs
http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
index 58761d5..aec6ca5 100644
--- a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
@@ -99,4 +99,10 @@ public class ATSConstants {
"yarn.timeline-service.webapp.address";
public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS_CONF_NAME =
"yarn.timeline-service.webapp.https.address";
+
+ public static final String TEZ_VERSION = "tezVersion";
+ public static final String VERSION = "version";
+ public static final String REVISION = "revision";
+ public static final String BUILD_TIME = "buildTime";
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java b/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
index b5e44c9..7e2a7e5 100644
--- a/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
+++ b/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.ClassUtil;
+import com.google.common.annotations.VisibleForTesting;
+
@Private
@Unstable
public class VersionInfo {
@@ -62,6 +64,18 @@ public class VersionInfo {
}
}
+ @VisibleForTesting
+ @Private
+ protected VersionInfo(String component, String version, String revision,
+ String buildTime, String scmUrl) {
+ this.info = new Properties();
+ this.component = component;
+ info.setProperty(VERSION, version);
+ info.setProperty(REVISION, revision);
+ info.setProperty(BUILD_TIME, buildTime);
+ info.setProperty(SCM_URL, scmUrl);
+ }
+
public String getVersion() {
return info.getProperty(VERSION, UNKNOWN);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 789de24..c47d093 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -462,7 +462,8 @@ public class DAGAppMaster extends AbstractService {
if (!versionMismatch) {
if (this.appAttemptID.getAttemptId() == 1) {
AppLaunchedEvent appLaunchedEvent = new AppLaunchedEvent(appAttemptID.getApplicationId(),
- startTime, appSubmitTime, appMasterUgi.getShortUserName(), this.amConf);
+ startTime, appSubmitTime, appMasterUgi.getShortUserName(), this.amConf,
+ dagVersionInfo);
historyEventHandler.handle(
new DAGHistoryEvent(appLaunchedEvent));
}
http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
index 4c79c53..08d2aff 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
@@ -23,8 +23,8 @@ import java.io.InputStream;
import java.io.OutputStream;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.common.VersionInfo;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.HistoryEventType;
@@ -35,18 +35,20 @@ public class AppLaunchedEvent implements HistoryEvent {
private long appSubmitTime;
private String user;
private Configuration conf;
+ private VersionInfo version;
public AppLaunchedEvent() {
}
public AppLaunchedEvent(ApplicationId appId,
long launchTime, long appSubmitTime, String user,
- Configuration conf) {
+ Configuration conf, VersionInfo version) {
this.applicationId = appId;
this.launchTime = launchTime;
this.appSubmitTime = appSubmitTime;
this.user = user;
this.conf = conf;
+ this.version = version;
}
@Override
@@ -101,4 +103,8 @@ public class AppLaunchedEvent implements HistoryEvent {
return conf;
}
+ public VersionInfo getVersion() {
+ return version;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index 5f259989..0f34811 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -30,6 +30,7 @@ import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.common.ATSConstants;
+import org.apache.tez.common.VersionInfo;
import org.apache.tez.common.counters.CounterGroup;
import org.apache.tez.common.counters.TezCounter;
import org.apache.tez.common.counters.TezCounters;
@@ -371,4 +372,13 @@ public class DAGUtils {
return atsConf;
}
+ public static Map<String, Object> convertTezVersionToATSMap(VersionInfo versionInfo) {
+ Map<String, Object> atsInfo = new TreeMap<String, Object>();
+ atsInfo.put(ATSConstants.VERSION, versionInfo.getVersion());
+ atsInfo.put(ATSConstants.BUILD_TIME, versionInfo.getBuildTime());
+ atsInfo.put(ATSConstants.REVISION, versionInfo.getRevision());
+ return atsInfo;
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
index a7a23db..a0c3231 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
@@ -119,7 +119,7 @@ public class TestHistoryEventsProtoConversion {
private void testAppLaunchedEvent() throws Exception {
AppLaunchedEvent event = new AppLaunchedEvent(ApplicationId.newInstance(0, 1),
- 100, 100, null, new Configuration(false));
+ 100, 100, null, new Configuration(false), null);
try {
testProtoConversion(event);
fail("Expected to fail on conversion");
http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
index c3d51c3..7b30af9 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
@@ -105,7 +105,7 @@ public class TestHistoryEventJsonConversion {
switch (eventType) {
case APP_LAUNCHED:
event = new AppLaunchedEvent(applicationId, random.nextInt(), random.nextInt(),
- user, new Configuration(false));
+ user, new Configuration(false), null);
break;
case AM_LAUNCHED:
event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(),
http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
index 19e2741..3e355e6 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
@@ -59,7 +59,7 @@ public class HistoryEventTimelineConversion {
throw new UnsupportedOperationException("Invalid Event, does not support history"
+ ", eventType=" + historyEvent.getEventType());
}
- TimelineEntity timelineEntity = null;
+ TimelineEntity timelineEntity;
switch (historyEvent.getEventType()) {
case APP_LAUNCHED:
timelineEntity = convertAppLaunchedEvent((AppLaunchedEvent) historyEvent);
@@ -144,6 +144,11 @@ public class HistoryEventTimelineConversion {
atsEntity.setStartTime(event.getLaunchTime());
+ if (event.getVersion() != null) {
+ atsEntity.addOtherInfo(ATSConstants.TEZ_VERSION,
+ DAGUtils.convertTezVersionToATSMap(event.getVersion()));
+ }
+
return atsEntity;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/6607e944/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index fe3d780..f32704e 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.tez.common.ATSConstants;
+import org.apache.tez.common.VersionInfo;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
@@ -108,7 +109,7 @@ public class TestHistoryEventTimelineConversion {
switch (eventType) {
case APP_LAUNCHED:
event = new AppLaunchedEvent(applicationId, random.nextInt(), random.nextInt(),
- user, new Configuration(false));
+ user, new Configuration(false), null);
break;
case AM_LAUNCHED:
event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(),
@@ -193,6 +194,14 @@ public class TestHistoryEventTimelineConversion {
}
}
+ static class MockVersionInfo extends VersionInfo {
+
+ MockVersionInfo() {
+ super("component", "1.1.0", "rev1", "20120101", "git.apache.org");
+ }
+
+ }
+
@Test
public void testConvertAppLaunchedEvent() {
long launchTime = random.nextLong();
@@ -201,9 +210,9 @@ public class TestHistoryEventTimelineConversion {
conf.set("foo", "bar");
conf.set("applicationId", "1234");
-
+ MockVersionInfo mockVersionInfo = new MockVersionInfo();
AppLaunchedEvent event = new AppLaunchedEvent(applicationId, launchTime,
- submitTime, user, conf);
+ submitTime, user, conf, mockVersionInfo);
TimelineEntity timelineEntity = HistoryEventTimelineConversion.convertToTimelineEntity(event);
@@ -221,13 +230,24 @@ public class TestHistoryEventTimelineConversion {
Assert.assertEquals(1, timelineEntity.getPrimaryFilters().size());
Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.USER).contains(user));
- Assert.assertEquals(1, timelineEntity.getOtherInfo().size());
+ Assert.assertEquals(2, timelineEntity.getOtherInfo().size());
Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.CONFIG));
+ Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.TEZ_VERSION));
Map<String, String> config =
(Map<String, String>)timelineEntity.getOtherInfo().get(ATSConstants.CONFIG);
Assert.assertEquals(conf.get("foo"), config.get("foo"));
Assert.assertEquals(conf.get("applicationId"), config.get("applicationId"));
+
+ Map<String, String> versionInfo =
+ (Map<String, String>)timelineEntity.getOtherInfo().get(ATSConstants.TEZ_VERSION);
+ Assert.assertEquals(mockVersionInfo.getVersion(),
+ versionInfo.get(ATSConstants.VERSION));
+ Assert.assertEquals(mockVersionInfo.getRevision(),
+ versionInfo.get(ATSConstants.REVISION));
+ Assert.assertEquals(mockVersionInfo.getBuildTime(),
+ versionInfo.get(ATSConstants.BUILD_TIME));
+
}
@Test(timeout = 5000)