You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2015/10/12 19:10:41 UTC
[01/50] [abbrv] hadoop git commit: MAPREDUCE-6327. Made MR AM use
timeline service v2 API to write history events and counters. Contributed by
Junping Du.
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928-rebase [created] bd5af9c5f
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index eab9026..b3ea26e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -18,22 +18,45 @@
package org.apache.hadoop.mapred;
+import java.io.File;
+import java.io.IOException;
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.jobhistory.EventType;
import org.apache.hadoop.mapreduce.jobhistory.TestJobHistoryEventHandler;
import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Assert;
import org.junit.Test;
public class TestMRTimelineEventHandling {
+ private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
+ private static final Log LOG =
+ LogFactory.getLog(TestMRTimelineEventHandling.class);
+
@Test
public void testTimelineServiceStartInMiniCluster() throws Exception {
Configuration conf = new YarnConfiguration();
@@ -47,7 +70,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestMRTimelineEventHandling.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
@@ -88,7 +111,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestMRTimelineEventHandling.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -132,6 +155,140 @@ public class TestMRTimelineEventHandling {
}
}
}
+
+ @Test
+ public void testMRNewTimelineServiceEventHandling() throws Exception {
+ LOG.info("testMRNewTimelineServiceEventHandling start.");
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
+
+ // enable new timeline serivce in MR side
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED, true);
+
+ // enable aux-service based timeline collectors
+ conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
+ conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
+ + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
+
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+
+ MiniMRYarnCluster cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestMRTimelineEventHandling.class.getSimpleName(), 1, true);
+ cluster.init(conf);
+ cluster.start();
+ LOG.info("A MiniMRYarnCluster get start.");
+
+ Path inDir = new Path("input");
+ Path outDir = new Path("output");
+ LOG.info("Run 1st job which should be successful.");
+ RunningJob job =
+ UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
+ Assert.assertEquals(JobStatus.SUCCEEDED,
+ job.getJobStatus().getState().getValue());
+
+ YarnClient yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(new Configuration(cluster.getConfig()));
+ yarnClient.start();
+ EnumSet<YarnApplicationState> appStates =
+ EnumSet.allOf(YarnApplicationState.class);
+
+ ApplicationId firstAppId = null;
+ List<ApplicationReport> apps = yarnClient.getApplications(appStates);
+ Assert.assertEquals(apps.size(), 1);
+ ApplicationReport appReport = apps.get(0);
+ firstAppId = appReport.getApplicationId();
+
+ checkNewTimelineEvent(firstAppId);
+
+ LOG.info("Run 2nd job which should be failed.");
+ job = UtilsForTests.runJobFail(new JobConf(conf), inDir, outDir);
+ Assert.assertEquals(JobStatus.FAILED,
+ job.getJobStatus().getState().getValue());
+
+ apps = yarnClient.getApplications(appStates);
+ Assert.assertEquals(apps.size(), 2);
+
+ ApplicationId secAppId = null;
+ secAppId = apps.get(0).getApplicationId() == firstAppId ?
+ apps.get(1).getApplicationId() : apps.get(0).getApplicationId();
+ checkNewTimelineEvent(firstAppId);
+
+ } finally {
+ if (cluster != null) {
+ cluster.stop();
+ }
+ // Cleanup test file
+ String testRoot =
+ FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT;
+ File testRootFolder = new File(testRoot);
+ if(testRootFolder.isDirectory()) {
+ FileUtils.deleteDirectory(testRootFolder);
+ }
+
+ }
+ }
+
+ private void checkNewTimelineEvent(ApplicationId appId) throws IOException {
+ String tmpRoot =
+ FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
+ + "/entities/";
+
+ File tmpRootFolder = new File(tmpRoot);
+
+ Assert.assertTrue(tmpRootFolder.isDirectory());
+ String basePath = tmpRoot + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
+ UserGroupInformation.getCurrentUser().getShortUserName() +
+ "/" + TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId) +
+ "/1/1/" + appId.toString();
+ // for this test, we expect MAPREDUCE_JOB and MAPREDUCE_TASK dirs
+ String outputDirJob = basePath + "/MAPREDUCE_JOB/";
+
+ File entityFolder = new File(outputDirJob);
+ Assert.assertTrue("Job output directory: " + outputDirJob + " is not exist.",
+ entityFolder.isDirectory());
+
+ // check for job event file
+ String jobEventFileName = appId.toString().replaceAll("application", "job")
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+
+ String jobEventFilePath = outputDirJob + jobEventFileName;
+ File jobEventFile = new File(jobEventFilePath);
+ Assert.assertTrue("jobEventFilePath: " + jobEventFilePath + " is not exist.",
+ jobEventFile.exists());
+
+ // check for task event file
+ String outputDirTask = basePath + "/MAPREDUCE_TASK/";
+ File taskFolder = new File(outputDirTask);
+ Assert.assertTrue("Task output directory: " + outputDirTask + " is not exist.",
+ taskFolder.isDirectory());
+
+ String taskEventFileName = appId.toString().replaceAll("application", "task")
+ + "_m_000000" + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+
+ String taskEventFilePath = outputDirTask + taskEventFileName;
+ File taskEventFile = new File(taskEventFilePath);
+ Assert.assertTrue("taskEventFileName: " + taskEventFilePath + " is not exist.",
+ taskEventFile.exists());
+
+ // check for task attempt event file
+ String outputDirTaskAttempt = basePath + "/MAPREDUCE_TASK_ATTEMPT/";
+ File taskAttemptFolder = new File(outputDirTaskAttempt);
+ Assert.assertTrue("TaskAttempt output directory: " + outputDirTaskAttempt +
+ " is not exist.", taskAttemptFolder.isDirectory());
+
+ String taskAttemptEventFileName = appId.toString().replaceAll(
+ "application", "attempt") + "_m_000000_0" +
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+
+ String taskAttemptEventFilePath = outputDirTaskAttempt +
+ taskAttemptEventFileName;
+ File taskAttemptEventFile = new File(taskAttemptEventFilePath);
+ Assert.assertTrue("taskAttemptEventFileName: " + taskAttemptEventFilePath +
+ " is not exist.", taskAttemptEventFile.exists());
+ }
@Test
public void testMapreduceJobTimelineServiceEnabled()
@@ -142,7 +299,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestMRTimelineEventHandling.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 47b38a1..90732eb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -66,6 +66,7 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
private static final Log LOG = LogFactory.getLog(MiniMRYarnCluster.class);
private JobHistoryServer historyServer;
private JobHistoryServerWrapper historyServerWrapper;
+ private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
public MiniMRYarnCluster(String testName) {
this(testName, 1);
@@ -167,8 +168,24 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
// which shuffle doesn't happen
//configure the shuffle service in NM
- conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
- new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
+ String[] nmAuxServices = conf.getStrings(YarnConfiguration.NM_AUX_SERVICES);
+ // if need to enable TIMELINE_AUX_SERVICE_NAME
+ boolean enableTimelineAuxService = false;
+ if (nmAuxServices != null) {
+ for (String nmAuxService: nmAuxServices) {
+ if (nmAuxService == TIMELINE_AUX_SERVICE_NAME) {
+ enableTimelineAuxService = true;
+ break;
+ }
+ }
+ }
+ if (enableTimelineAuxService) {
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
+ new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, TIMELINE_AUX_SERVICE_NAME });
+ } else {
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
+ new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
+ }
conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
Service.class);
[12/50] [abbrv] hadoop git commit: YARN-3276. Code cleanup for
timeline service API records. Contributed by Junping Du.
Posted by sj...@apache.org.
YARN-3276. Code cleanup for timeline service API records. Contributed by Junping Du.
(cherry picked from commit d88f30ba5359f59fb71b93a55e1c1d9a1c0dff8e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec0fd03f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec0fd03f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec0fd03f
Branch: refs/heads/YARN-2928-rebase
Commit: ec0fd03f0e642f398cf2e5fcce6602070a6b4362
Parents: 5932863
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Jun 3 15:13:29 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:56:41 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../api/records/timeline/TimelineEntity.java | 21 ++----
.../api/records/timeline/TimelineEvent.java | 8 +--
.../records/timelineservice/TimelineEntity.java | 33 ++-------
.../records/timelineservice/TimelineEvent.java | 7 +-
.../records/timelineservice/TimelineMetric.java | 2 +-
.../hadoop/yarn/util/TimelineServiceHelper.java | 47 ++++++++++++
.../impl/pb/AllocateResponsePBImpl.java | 4 +-
.../yarn/util/TestTimelineServiceHelper.java | 76 ++++++++++++++++++++
9 files changed, 147 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0fd03f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e56403d..8ff151a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -84,6 +84,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
IMPROVEMENTS
+ YARN-3276. Code cleanup for timeline service API records. (Junping Du via
+ zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0fd03f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
index a43259b..e695050 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
@@ -34,6 +34,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
/**
* <p>
@@ -231,11 +232,8 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
*/
public void setRelatedEntities(
Map<String, Set<String>> relatedEntities) {
- if (relatedEntities != null && !(relatedEntities instanceof HashMap)) {
- this.relatedEntities = new HashMap<String, Set<String>>(relatedEntities);
- } else {
- this.relatedEntities = (HashMap<String, Set<String>>) relatedEntities;
- }
+ this.relatedEntities = TimelineServiceHelper.mapCastToHashMap(
+ relatedEntities);
}
/**
@@ -297,11 +295,8 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
* a map of primary filters
*/
public void setPrimaryFilters(Map<String, Set<Object>> primaryFilters) {
- if (primaryFilters != null && !(primaryFilters instanceof HashMap)) {
- this.primaryFilters = new HashMap<String, Set<Object>>(primaryFilters);
- } else {
- this.primaryFilters = (HashMap<String, Set<Object>>) primaryFilters;
- }
+ this.primaryFilters =
+ TimelineServiceHelper.mapCastToHashMap(primaryFilters);
}
/**
@@ -350,11 +345,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
* a map of other information
*/
public void setOtherInfo(Map<String, Object> otherInfo) {
- if (otherInfo != null && !(otherInfo instanceof HashMap)) {
- this.otherInfo = new HashMap<String, Object>(otherInfo);
- } else {
- this.otherInfo = (HashMap<String, Object>) otherInfo;
- }
+ this.otherInfo = TimelineServiceHelper.mapCastToHashMap(otherInfo);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0fd03f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
index 73b2e72..d5611f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
@@ -29,6 +29,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
/**
* The class that contains the information of an event that is related to some
@@ -135,11 +136,8 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
* a map of of the information of the event
*/
public void setEventInfo(Map<String, Object> eventInfo) {
- if (eventInfo != null && !(eventInfo instanceof HashMap)) {
- this.eventInfo = new HashMap<String, Object>(eventInfo);
- } else {
- this.eventInfo = (HashMap<String, Object>) eventInfo;
- }
+ this.eventInfo = TimelineServiceHelper.mapCastToHashMap(
+ eventInfo);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0fd03f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 3be7f52..defadec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -184,11 +185,7 @@ public class TimelineEntity {
public void setInfo(Map<String, Object> info) {
if (real == null) {
- if (info != null && !(info instanceof HashMap)) {
- this.info = new HashMap<String, Object>(info);
- } else {
- this.info = (HashMap<String, Object>) info;
- }
+ this.info = TimelineServiceHelper.mapCastToHashMap(info);
} else {
real.setInfo(info);
}
@@ -231,11 +228,7 @@ public class TimelineEntity {
public void setConfigs(Map<String, String> configs) {
if (real == null) {
- if (configs != null && !(configs instanceof HashMap)) {
- this.configs = new HashMap<String, String>(configs);
- } else {
- this.configs = (HashMap<String, String>) configs;
- }
+ this.configs = TimelineServiceHelper.mapCastToHashMap(configs);
} else {
real.setConfigs(configs);
}
@@ -345,14 +338,8 @@ public class TimelineEntity {
public void setIsRelatedToEntities(
Map<String, Set<String>> isRelatedToEntities) {
if (real == null) {
- if (isRelatedToEntities != null &&
- !(isRelatedToEntities instanceof HashMap)) {
- this.isRelatedToEntities =
- new HashMap<String, Set<String>>(isRelatedToEntities);
- } else {
- this.isRelatedToEntities =
- (HashMap<String, Set<String>>) isRelatedToEntities;
- }
+ this.isRelatedToEntities =
+ TimelineServiceHelper.mapCastToHashMap(isRelatedToEntities);
} else {
real.setIsRelatedToEntities(isRelatedToEntities);
}
@@ -438,14 +425,8 @@ public class TimelineEntity {
public void setRelatesToEntities(Map<String, Set<String>> relatesToEntities) {
if (real == null) {
- if (relatesToEntities != null &&
- !(relatesToEntities instanceof HashMap)) {
- this.relatesToEntities =
- new HashMap<String, Set<String>>(relatesToEntities);
- } else {
- this.relatesToEntities =
- (HashMap<String, Set<String>>) relatesToEntities;
- }
+ this.relatesToEntities =
+ TimelineServiceHelper.mapCastToHashMap(relatesToEntities);
} else {
real.setRelatesToEntities(relatesToEntities);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0fd03f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
index 517c88c..b6b82a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -61,11 +62,7 @@ public class TimelineEvent {
}
public void setInfo(Map<String, Object> info) {
- if (info != null && !(info instanceof HashMap)) {
- this.info = new HashMap<String, Object>(info);
- } else {
- this.info = (HashMap<String, Object>) info;
- }
+ this.info = TimelineServiceHelper.mapCastToHashMap(info);
}
public void addInfo(Map<String, Object> info) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0fd03f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index c897d39..172b56c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -44,7 +44,7 @@ public class TimelineMetric {
private Comparator<Long> reverseComparator = new Comparator<Long>() {
@Override
public int compare(Long l1, Long l2) {
- return -l1.compareTo(l2);
+ return l2.compareTo(l1);
}
};
private TreeMap<Long, Number> values = new TreeMap<>(reverseComparator);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0fd03f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java
new file mode 100644
index 0000000..ff6ebbd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+
+/**
+ * Helper class for Timeline service.
+ */
+@LimitedPrivate({ "MapReduce", "YARN" })
+public final class TimelineServiceHelper {
+
+ private TimelineServiceHelper() {
+ // Utility classes should not have a public or default constructor.
+ }
+
+ /**
+ * Cast map to HashMap for generic type.
+ * @param originalMap the map need to be casted
+ * @return casted HashMap object
+ */
+ public static <E, V> HashMap<E, V> mapCastToHashMap(
+ Map<E, V> originalMap) {
+ return originalMap == null ? null : originalMap instanceof HashMap ?
+ (HashMap<E, V>) originalMap : new HashMap<E, V>(originalMap);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0fd03f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
index d1e00f5..f5801e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
@@ -380,13 +380,13 @@ public class AllocateResponsePBImpl extends AllocateResponse {
@Override
- public String getCollectorAddr() {
+ public synchronized String getCollectorAddr() {
AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
return p.getCollectorAddr();
}
@Override
- public void setCollectorAddr(String collectorAddr) {
+ public synchronized void setCollectorAddr(String collectorAddr) {
maybeInitBuilder();
if (collectorAddr == null) {
builder.clearCollectorAddr();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0fd03f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java
new file mode 100644
index 0000000..f852df0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.util;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestTimelineServiceHelper {
+
+ @Test
+ public void testMapCastToHashMap() {
+
+ // Test null map be casted to null
+ Map<String, String> nullMap = null;
+ Assert.assertNull(TimelineServiceHelper.mapCastToHashMap(nullMap));
+
+ // Test empty hashmap be casted to a empty hashmap
+ Map<String, String> emptyHashMap = new HashMap<String, String>();
+ Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(emptyHashMap).size(), 0);
+
+ // Test empty non-hashmap be casted to a empty hashmap
+ Map<String, String> emptyTreeMap = new TreeMap<String, String>();
+ Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(emptyTreeMap).size(), 0);
+
+ // Test non-empty hashmap be casted to hashmap correctly
+ Map<String, String> firstHashMap = new HashMap<String, String>();
+ String key = "KEY";
+ String value = "VALUE";
+ firstHashMap.put(key, value);
+ Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(firstHashMap), firstHashMap);
+
+ // Test non-empty non-hashmap is casted correctly.
+ Map<String, String> firstTreeMap = new TreeMap<String, String>();
+ firstTreeMap.put(key, value);
+ HashMap<String, String> alternateHashMap =
+ TimelineServiceHelper.mapCastToHashMap(firstTreeMap);
+ Assert.assertEquals(firstTreeMap.size(), alternateHashMap.size());
+ Assert.assertEquals(alternateHashMap.get(key), value);
+
+ // Test complicated hashmap be casted correctly
+ Map<String, Set<String>> complicatedHashMap = new HashMap<String, Set<String>>();
+ Set<String> hashSet = new HashSet<String>();
+ hashSet.add(value);
+ complicatedHashMap.put(key, hashSet);
+ Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedHashMap),
+ complicatedHashMap);
+
+ // Test complicated non-hashmap get casted correctly
+ Map<String, Set<String>> complicatedTreeMap = new TreeMap<String, Set<String>>();
+ complicatedTreeMap.put(key, hashSet);
+ Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedTreeMap).get(key),
+ hashSet);
+ }
+
+}
[21/50] [abbrv] hadoop git commit: YARN-3836. add equals and hashCode
to TimelineEntity and other classes in the data model (Li Lu via sjlee)
Posted by sj...@apache.org.
YARN-3836. add equals and hashCode to TimelineEntity and other classes in the data model (Li Lu via sjlee)
(cherry picked from commit 2d4a8f4563c06339717ca9410b2794754603fba3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95e76070
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95e76070
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95e76070
Branch: refs/heads/YARN-2928-rebase
Commit: 95e7607083e72dabb877283ebbb226d93e7deaba
Parents: 6f0e7827
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu Jul 9 20:50:48 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:38 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/timelineservice/TimelineEntity.java | 89 +++++++++++++++++++-
.../records/timelineservice/TimelineEvent.java | 41 ++++++++-
.../records/timelineservice/TimelineMetric.java | 30 +++++++
.../TestTimelineServiceRecords.java | 36 +++++++-
5 files changed, 195 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95e76070/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e391cf9..8542191 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -87,6 +87,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3706. Generalize native HBase writer for additional tables (Joep
Rottinghuis via sjlee)
+ YARN-3836. add equals and hashCode to TimelineEntity and other classes in
+ the data model (Li Lu via sjlee)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95e76070/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 60fba85..9ef2d90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -31,11 +31,25 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+/**
+ * The basic timeline entity data structure for timeline service v2. Timeline
+ * entity objects are not thread safe and should not be accessed concurrently.
+ * All collection members will be initialized into empty collections. Two
+ * timeline entities are equal iff. their type and id are identical.
+ *
+ * All non-primitive type, non-collection members will be initialized into null.
+ * User should set the type and id of a timeline entity to make it valid (can be
+ * checked by using the {@link #isValid()} method). Callers to the getters
+ * should perform null checks for non-primitive type, non-collection members.
+ *
+ * Callers are recommended not to alter the returned collection objects from the
+ * getters.
+ */
@XmlRootElement(name = "entity")
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public class TimelineEntity {
+public class TimelineEntity implements Comparable<TimelineEntity> {
protected final static String SYSTEM_INFO_KEY_PREFIX = "SYSTEM_INFO_";
@XmlRootElement(name = "identifier")
@@ -77,6 +91,41 @@ public class TimelineEntity {
"type='" + type + '\'' +
", id='" + id + '\'' + "]";
}
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((id == null) ? 0 : id.hashCode());
+ result =
+ prime * result + ((type == null) ? 0 : type.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (!(obj instanceof Identifier)) {
+ return false;
+ }
+ Identifier other = (Identifier) obj;
+ if (id == null) {
+ if (other.getId() != null) {
+ return false;
+ }
+ } else if (!id.equals(other.getId())) {
+ return false;
+ }
+ if (type == null) {
+ if (other.getType() != null) {
+ return false;
+ }
+ } else if (!type.equals(other.getType())) {
+ return false;
+ }
+ return true;
+ }
}
private TimelineEntity real;
@@ -471,6 +520,44 @@ public class TimelineEntity {
}
}
+ public boolean isValid() {
+ return (getId() != null && getType() != null);
+ }
+
+ // When get hashCode for a timeline entity, or check if two timeline entities
+ // are equal, we only compare their identifiers (id and type)
+ @Override
+ public int hashCode() {
+ return getIdentifier().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (!(obj instanceof TimelineEntity))
+ return false;
+ TimelineEntity other = (TimelineEntity) obj;
+ return getIdentifier().equals(other.getIdentifier());
+ }
+
+ @Override
+ public int compareTo(TimelineEntity other) {
+ int comparison = getType().compareTo(other.getType());
+ if (comparison == 0) {
+ if (getCreatedTime() > other.getCreatedTime()) {
+ // Order by created time desc
+ return -1;
+ } else if (getCreatedTime() < other.getCreatedTime()) {
+ return 1;
+ } else {
+ return getId().compareTo(other.getId());
+ }
+ } else {
+ return comparison;
+ }
+ }
+
protected TimelineEntity getReal() {
return real == null ? this : real;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95e76070/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
index b6b82a7..1dbf7e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
@@ -32,7 +32,7 @@ import java.util.Map;
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public class TimelineEvent {
+public class TimelineEvent implements Comparable<TimelineEvent> {
private String id;
private HashMap<String, Object> info = new HashMap<>();
private long timestamp;
@@ -81,4 +81,43 @@ public class TimelineEvent {
public void setTimestamp(long timestamp) {
this.timestamp = timestamp;
}
+
+ public boolean isValid() {
+ return (id != null && timestamp != 0L);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = (int) (timestamp ^ (timestamp >>> 32));
+ result = 31 * result + id.hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (!(o instanceof TimelineEvent))
+ return false;
+
+ TimelineEvent event = (TimelineEvent) o;
+
+ if (timestamp != event.timestamp)
+ return false;
+ if (!id.equals(event.id)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int compareTo(TimelineEvent other) {
+ if (timestamp > other.timestamp) {
+ return -1;
+ } else if (timestamp < other.timestamp) {
+ return 1;
+ } else {
+ return id.compareTo(other.id);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95e76070/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index 172b56c..f44c1a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -124,4 +124,34 @@ public class TimelineMetric {
this.values.clear();
this.values.putAll(values);
}
+
+ public boolean isValid() {
+ return (id != null);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = id.hashCode();
+ result = 31 * result + type.hashCode();
+ return result;
+ }
+
+ // Only check if type and id are equal
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (!(o instanceof TimelineMetric))
+ return false;
+
+ TimelineMetric m = (TimelineMetric) o;
+
+ if (!id.equals(m.id)) {
+ return false;
+ }
+ if (type != m.type) {
+ return false;
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95e76070/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index 95b922e..78943e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -100,6 +100,13 @@ public class TestTimelineServiceRecords {
}
entity.addMetric(metric2);
+ TimelineMetric metric3 = new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
+ metric3.setId("test metric id 1");
+ metric3.addValue(4L, (short) 4);
+ Assert.assertEquals("metric3 should equal to metric2! ", metric3, metric2);
+ Assert.assertNotEquals("metric1 should not equal to metric2! ",
+ metric1, metric2);
+
TimelineEvent event1 = new TimelineEvent();
event1.setId("test event id 1");
event1.addInfo("test info key 1", "test info value 1");
@@ -108,7 +115,7 @@ public class TestTimelineServiceRecords {
event1.addInfo("test info key 3", true);
Assert.assertTrue(
event1.getInfo().get("test info key 3") instanceof Boolean);
- event1.setTimestamp(0L);
+ event1.setTimestamp(1L);
entity.addEvent(event1);
TimelineEvent event2 = new TimelineEvent();
@@ -119,9 +126,18 @@ public class TestTimelineServiceRecords {
event2.addInfo("test info key 3", true);
Assert.assertTrue(
event2.getInfo().get("test info key 3") instanceof Boolean);
- event2.setTimestamp(1L);
+ event2.setTimestamp(2L);
entity.addEvent(event2);
+ Assert.assertFalse("event1 should not equal to event2! ",
+ event1.equals(event2));
+ TimelineEvent event3 = new TimelineEvent();
+ event3.setId("test event id 1");
+ event3.setTimestamp(1L);
+ Assert.assertEquals("event1 should equal to event3! ", event3, event1);
+ Assert.assertNotEquals("event1 should not equal to event2! ",
+ event1, event2);
+
entity.setCreatedTime(0L);
entity.setModifiedTime(1L);
entity.addRelatesToEntity("test type 2", "test id 2");
@@ -136,6 +152,22 @@ public class TestTimelineServiceRecords {
TimelineEntity entity2 = new TimelineEntity();
entities.addEntity(entity2);
LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(entities, true));
+
+ Assert.assertFalse("entity 1 should not be valid without type and id",
+ entity1.isValid());
+ entity1.setId("test id 2");
+ entity1.setType("test type 2");
+ entity2.setId("test id 1");
+ entity2.setType("test type 1");
+
+ Assert.assertEquals("Timeline entity should equal to entity2! ",
+ entity, entity2);
+ Assert.assertNotEquals("entity1 should not equal to entity! ",
+ entity1, entity);
+ Assert.assertEquals("entity should be less than entity1! ",
+ entity1.compareTo(entity), 1);
+ Assert.assertEquals("entity's hash code should be -28727840 but not "
+ + entity.hashCode(), entity.hashCode(), -28727840);
}
@Test
[35/50] [abbrv] hadoop git commit: YARN-4064. build is broken at
TestHBaseTimelineWriterImpl.java (sjlee)
Posted by sj...@apache.org.
YARN-4064. build is broken at TestHBaseTimelineWriterImpl.java (sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a81fa64a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a81fa64a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a81fa64a
Branch: refs/heads/YARN-2928-rebase
Commit: a81fa64a2650f48b18ad51b14dc2910fdfd79f59
Parents: f98f97b
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed Aug 19 17:46:03 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 16:13:29 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../storage/TestHBaseTimelineStorage.java | 770 +++++++++++++++++++
.../storage/TestHBaseTimelineWriterImpl.java | 770 -------------------
3 files changed, 772 insertions(+), 770 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a81fa64a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d80ffa0..aa5dde2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -139,6 +139,8 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3984. Adjusted the event column key schema and avoided missing empty
event. (Vrushali C via zjshen)
+ YARN-4064. build is broken at TestHBaseTimelineWriterImpl.java (sjlee)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a81fa64a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
new file mode 100644
index 0000000..2875e01
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -0,0 +1,770 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Various tests to test writing entities to HBase and reading them back from
+ * it.
+ *
+ * It uses a single HBase mini-cluster for all tests which is a little more
+ * realistic, and helps test correctness in the presence of other data.
+ *
+ * Each test uses a different cluster name to be able to handle its own data
+ * even if other records exist in the table. Use a different cluster name if
+ * you add a new test.
+ */
+public class TestHBaseTimelineStorage {
+
+ private static HBaseTestingUtility util;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ util = new HBaseTestingUtility();
+ util.startMiniCluster();
+ createSchema();
+ }
+
+ private static void createSchema() throws IOException {
+ new EntityTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
+ new AppToFlowTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
+ new ApplicationTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
+ }
+
+ @Test
+ public void testWriteApplicationToHBase() throws Exception {
+ TimelineEntities te = new TimelineEntities();
+ ApplicationEntity entity = new ApplicationEntity();
+ String id = "hello";
+ entity.setId(id);
+ long cTime = 1425016501000L;
+ long mTime = 1425026901000L;
+ entity.setCreatedTime(cTime);
+ entity.setModifiedTime(mTime);
+
+ // add the info map in Timeline Entity
+ Map<String, Object> infoMap = new HashMap<String, Object>();
+ infoMap.put("infoMapKey1", "infoMapValue1");
+ infoMap.put("infoMapKey2", 10);
+ entity.addInfo(infoMap);
+
+ // add the isRelatedToEntity info
+ String key = "task";
+ String value = "is_related_to_entity_id_here";
+ Set<String> isRelatedToSet = new HashSet<String>();
+ isRelatedToSet.add(value);
+ Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
+ isRelatedTo.put(key, isRelatedToSet);
+ entity.setIsRelatedToEntities(isRelatedTo);
+
+ // add the relatesTo info
+ key = "container";
+ value = "relates_to_entity_id_here";
+ Set<String> relatesToSet = new HashSet<String>();
+ relatesToSet.add(value);
+ value = "relates_to_entity_id_here_Second";
+ relatesToSet.add(value);
+ Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
+ relatesTo.put(key, relatesToSet);
+ entity.setRelatesToEntities(relatesTo);
+
+ // add some config entries
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put("config_param1", "value1");
+ conf.put("config_param2", "value2");
+ entity.addConfigs(conf);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 120000, 100000000);
+ metricValues.put(ts - 100000, 200000000);
+ metricValues.put(ts - 80000, 300000000);
+ metricValues.put(ts - 60000, 400000000);
+ metricValues.put(ts - 40000, 50000000000L);
+ metricValues.put(ts - 20000, 60000000000L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ te.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ String cluster = "cluster_test_write_app";
+ String user = "user1";
+ String flow = "some_flow_name";
+ String flowVersion = "AB7822C10F1111";
+ long runid = 1002345678919L;
+ hbi.write(cluster, user, flow, flowVersion, runid, id, te);
+ hbi.stop();
+
+ // retrieve the row
+ byte[] rowKey =
+ ApplicationRowKey.getRowKey(cluster, user, flow, runid, id);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Result result = new ApplicationTable().getResult(c1, conn, get);
+
+ assertTrue(result != null);
+ assertEquals(16, result.size());
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
+ id));
+
+ // check info column family
+ String id1 = ApplicationColumn.ID.readResult(result).toString();
+ assertEquals(id, id1);
+
+ Number val =
+ (Number) ApplicationColumn.CREATED_TIME.readResult(result);
+ long cTime1 = val.longValue();
+ assertEquals(cTime1, cTime);
+
+ val = (Number) ApplicationColumn.MODIFIED_TIME.readResult(result);
+ long mTime1 = val.longValue();
+ assertEquals(mTime1, mTime);
+
+ Map<String, Object> infoColumns =
+ ApplicationColumnPrefix.INFO.readResults(result);
+ assertEquals(infoMap, infoColumns);
+
+ // Remember isRelatedTo is of type Map<String, Set<String>>
+ for (String isRelatedToKey : isRelatedTo.keySet()) {
+ Object isRelatedToValue =
+ ApplicationColumnPrefix.IS_RELATED_TO.readResult(result,
+ isRelatedToKey);
+ String compoundValue = isRelatedToValue.toString();
+ // id7?id9?id6
+ Set<String> isRelatedToValues =
+ new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(isRelatedTo.get(isRelatedToKey).size(),
+ isRelatedToValues.size());
+ for (String v : isRelatedTo.get(isRelatedToKey)) {
+ assertTrue(isRelatedToValues.contains(v));
+ }
+ }
+
+ // RelatesTo
+ for (String relatesToKey : relatesTo.keySet()) {
+ String compoundValue =
+ ApplicationColumnPrefix.RELATES_TO.readResult(result,
+ relatesToKey).toString();
+ // id3?id4?id5
+ Set<String> relatesToValues =
+ new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(relatesTo.get(relatesToKey).size(),
+ relatesToValues.size());
+ for (String v : relatesTo.get(relatesToKey)) {
+ assertTrue(relatesToValues.contains(v));
+ }
+ }
+
+ // Configuration
+ Map<String, Object> configColumns =
+ ApplicationColumnPrefix.CONFIG.readResults(result);
+ assertEquals(conf, configColumns);
+
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
+
+ NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
+ assertEquals(metricValues, metricMap);
+
+ // read the timeline entity using the reader this time
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, id,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ id, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
+
+ // verify attributes
+ assertEquals(id, e1.getId());
+ assertEquals(TimelineEntityType.YARN_APPLICATION.toString(),
+ e1.getType());
+ assertEquals(cTime, e1.getCreatedTime());
+ assertEquals(mTime, e1.getModifiedTime());
+ Map<String, Object> infoMap2 = e1.getInfo();
+ assertEquals(infoMap, infoMap2);
+
+ Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
+ assertEquals(isRelatedTo, isRelatedTo2);
+
+ Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
+ assertEquals(relatesTo, relatesTo2);
+
+ Map<String, String> conf2 = e1.getConfigs();
+ assertEquals(conf, conf2);
+
+ Set<TimelineMetric> metrics2 = e1.getMetrics();
+ assertEquals(metrics, metrics2);
+ for (TimelineMetric metric2 : metrics2) {
+ Map<Long, Number> metricValues2 = metric2.getValues();
+ assertEquals(metricValues, metricValues2);
+ }
+ } finally {
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
+ }
+ }
+
+ @Test
+ public void testWriteEntityToHBase() throws Exception {
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entity = new TimelineEntity();
+ String id = "hello";
+ String type = "world";
+ entity.setId(id);
+ entity.setType(type);
+ long cTime = 1425016501000L;
+ long mTime = 1425026901000L;
+ entity.setCreatedTime(cTime);
+ entity.setModifiedTime(mTime);
+
+ // add the info map in Timeline Entity
+ Map<String, Object> infoMap = new HashMap<String, Object>();
+ infoMap.put("infoMapKey1", "infoMapValue1");
+ infoMap.put("infoMapKey2", 10);
+ entity.addInfo(infoMap);
+
+ // add the isRelatedToEntity info
+ String key = "task";
+ String value = "is_related_to_entity_id_here";
+ Set<String> isRelatedToSet = new HashSet<String>();
+ isRelatedToSet.add(value);
+ Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
+ isRelatedTo.put(key, isRelatedToSet);
+ entity.setIsRelatedToEntities(isRelatedTo);
+
+ // add the relatesTo info
+ key = "container";
+ value = "relates_to_entity_id_here";
+ Set<String> relatesToSet = new HashSet<String>();
+ relatesToSet.add(value);
+ value = "relates_to_entity_id_here_Second";
+ relatesToSet.add(value);
+ Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
+ relatesTo.put(key, relatesToSet);
+ entity.setRelatesToEntities(relatesTo);
+
+ // add some config entries
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put("config_param1", "value1");
+ conf.put("config_param2", "value2");
+ entity.addConfigs(conf);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 120000, 100000000);
+ metricValues.put(ts - 100000, 200000000);
+ metricValues.put(ts - 80000, 300000000);
+ metricValues.put(ts - 60000, 400000000);
+ metricValues.put(ts - 40000, 50000000000L);
+ metricValues.put(ts - 20000, 60000000000L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ te.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ String cluster = "cluster_test_write_entity";
+ String user = "user1";
+ String flow = "some_flow_name";
+ String flowVersion = "AB7822C10F1111";
+ long runid = 1002345678919L;
+ String appName = "some app name";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+ hbi.stop();
+
+ // scan the table and see that entity exists
+ Scan s = new Scan();
+ byte[] startRow =
+ EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
+ s.setStartRow(startRow);
+ s.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+
+ int rowCount = 0;
+ int colCount = 0;
+ for (Result result : scanner) {
+ if (result != null && !result.isEmpty()) {
+ rowCount++;
+ colCount += result.size();
+ byte[] row1 = result.getRow();
+ assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
+ entity));
+
+ // check info column family
+ String id1 = EntityColumn.ID.readResult(result).toString();
+ assertEquals(id, id1);
+
+ String type1 = EntityColumn.TYPE.readResult(result).toString();
+ assertEquals(type, type1);
+
+ Number val = (Number) EntityColumn.CREATED_TIME.readResult(result);
+ long cTime1 = val.longValue();
+ assertEquals(cTime1, cTime);
+
+ val = (Number) EntityColumn.MODIFIED_TIME.readResult(result);
+ long mTime1 = val.longValue();
+ assertEquals(mTime1, mTime);
+
+ Map<String, Object> infoColumns =
+ EntityColumnPrefix.INFO.readResults(result);
+ assertEquals(infoMap, infoColumns);
+
+ // Remember isRelatedTo is of type Map<String, Set<String>>
+ for (String isRelatedToKey : isRelatedTo.keySet()) {
+ Object isRelatedToValue =
+ EntityColumnPrefix.IS_RELATED_TO.readResult(result,
+ isRelatedToKey);
+ String compoundValue = isRelatedToValue.toString();
+ // id7?id9?id6
+ Set<String> isRelatedToValues =
+ new HashSet<String>(
+ Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(isRelatedTo.get(isRelatedToKey).size(),
+ isRelatedToValues.size());
+ for (String v : isRelatedTo.get(isRelatedToKey)) {
+ assertTrue(isRelatedToValues.contains(v));
+ }
+ }
+
+ // RelatesTo
+ for (String relatesToKey : relatesTo.keySet()) {
+ String compoundValue =
+ EntityColumnPrefix.RELATES_TO.readResult(result, relatesToKey)
+ .toString();
+ // id3?id4?id5
+ Set<String> relatesToValues =
+ new HashSet<String>(
+ Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(relatesTo.get(relatesToKey).size(),
+ relatesToValues.size());
+ for (String v : relatesTo.get(relatesToKey)) {
+ assertTrue(relatesToValues.contains(v));
+ }
+ }
+
+ // Configuration
+ Map<String, Object> configColumns =
+ EntityColumnPrefix.CONFIG.readResults(result);
+ assertEquals(conf, configColumns);
+
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
+
+ NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
+ assertEquals(metricValues, metricMap);
+ }
+ }
+ assertEquals(1, rowCount);
+ assertEquals(17, colCount);
+
+ // read the timeline entity using the reader this time
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
+
+ // verify attributes
+ assertEquals(id, e1.getId());
+ assertEquals(type, e1.getType());
+ assertEquals(cTime, e1.getCreatedTime());
+ assertEquals(mTime, e1.getModifiedTime());
+ Map<String, Object> infoMap2 = e1.getInfo();
+ assertEquals(infoMap, infoMap2);
+
+ Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
+ assertEquals(isRelatedTo, isRelatedTo2);
+
+ Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
+ assertEquals(relatesTo, relatesTo2);
+
+ Map<String, String> conf2 = e1.getConfigs();
+ assertEquals(conf, conf2);
+
+ Set<TimelineMetric> metrics2 = e1.getMetrics();
+ assertEquals(metrics, metrics2);
+ for (TimelineMetric metric2 : metrics2) {
+ Map<Long, Number> metricValues2 = metric2.getValues();
+ assertEquals(metricValues, metricValues2);
+ }
+ } finally {
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
+ }
+ }
+
+ private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
+ String flow, long runid, String appName, TimelineEntity te) {
+
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
+
+ assertTrue(rowKeyComponents.length == 7);
+ assertEquals(user, Bytes.toString(rowKeyComponents[0]));
+ assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
+ assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
+ assertEquals(TimelineWriterUtils.invert(runid),
+ Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
+ assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
+ assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
+ return true;
+ }
+
+ private boolean isApplicationRowKeyCorrect(byte[] rowKey, String cluster,
+ String user, String flow, long runid, String appName) {
+
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
+
+ assertTrue(rowKeyComponents.length == 5);
+ assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
+ assertEquals(user, Bytes.toString(rowKeyComponents[1]));
+ assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
+ assertEquals(TimelineWriterUtils.invert(runid),
+ Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
+ return true;
+ }
+
+ @Test
+ public void testEvents() throws IOException {
+ TimelineEvent event = new TimelineEvent();
+ String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
+ event.setId(eventId);
+ long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+ String expKey = "foo_event";
+ Object expVal = "test";
+ event.addInfo(expKey, expVal);
+
+ final TimelineEntity entity = new ApplicationEntity();
+ entity.setId(ApplicationId.newInstance(0, 1).toString());
+ entity.addEvent(event);
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ String cluster = "cluster_test_events";
+ String user = "user2";
+ String flow = "other_flow_name";
+ String flowVersion = "1111F01C2287BA";
+ long runid = 1009876543218L;
+ String appName = "some app name";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
+ hbi.stop();
+
+ // retrieve the row
+ byte[] rowKey =
+ ApplicationRowKey.getRowKey(cluster, user, flow, runid, appName);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Result result = new ApplicationTable().getResult(c1, conn, get);
+
+ assertTrue(result != null);
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
+ appName));
+
+ Map<?, Object> eventsResult =
+ ApplicationColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
+ // there should be only one event
+ assertEquals(1, eventsResult.size());
+ for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
+ // the qualifier is a compound key
+ // hence match individual values
+ byte[][] karr = (byte[][])e.getKey();
+ assertEquals(3, karr.length);
+ assertEquals(eventId, Bytes.toString(karr[0]));
+ assertEquals(TimelineWriterUtils.invert(expTs), Bytes.toLong(karr[1]));
+ assertEquals(expKey, Bytes.toString(karr[2]));
+ Object value = e.getValue();
+ // there should be only one timestamp and value
+ assertEquals(expVal, value.toString());
+ }
+
+ // read the timeline entity using the reader this time
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ TimelineEntity e2 = hbr.getEntity(user, cluster, null, null, appName,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es2 = hbr.getEntities(user, cluster, null, null,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertNotNull(e2);
+ assertEquals(e1, e2);
+ assertEquals(1, es1.size());
+ assertEquals(1, es2.size());
+ assertEquals(es1, es2);
+
+ // check the events
+ NavigableSet<TimelineEvent> events = e1.getEvents();
+ // there should be only one event
+ assertEquals(1, events.size());
+ for (TimelineEvent e : events) {
+ assertEquals(eventId, e.getId());
+ assertEquals(expTs, e.getTimestamp());
+ Map<String,Object> info = e.getInfo();
+ assertEquals(1, info.size());
+ for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
+ assertEquals(expKey, infoEntry.getKey());
+ assertEquals(expVal, infoEntry.getValue());
+ }
+ }
+ } finally {
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
+ }
+ }
+
+ @Test
+ public void testEventsWithEmptyInfo() throws IOException {
+ TimelineEvent event = new TimelineEvent();
+ String eventId = "foo_event_id";
+ event.setId(eventId);
+ long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId("attempt_1329348432655_0001_m_000008_18");
+ entity.setType("FOO_ATTEMPT");
+ entity.addEvent(event);
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ String cluster = "cluster_test_empty_eventkey";
+ String user = "user_emptyeventkey";
+ String flow = "other_flow_name";
+ String flowVersion = "1111F01C2287BA";
+ long runid = 1009876543218L;
+ String appName = "some app name";
+ byte[] startRow =
+ EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
+ hbi.stop();
+ // scan the table and see that entity exists
+ Scan s = new Scan();
+ s.setStartRow(startRow);
+ s.addFamily(EntityColumnFamily.INFO.getBytes());
+ Connection conn = ConnectionFactory.createConnection(c1);
+ ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+
+ int rowCount = 0;
+ for (Result result : scanner) {
+ if (result != null && !result.isEmpty()) {
+ rowCount++;
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
+ entity));
+
+ Map<?, Object> eventsResult =
+ EntityColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
+ // there should be only one event
+ assertEquals(1, eventsResult.size());
+ for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
+ // the qualifier is a compound key
+ // hence match individual values
+ byte[][] karr = (byte[][])e.getKey();
+ assertEquals(3, karr.length);
+ assertEquals(eventId, Bytes.toString(karr[0]));
+ assertEquals(TimelineWriterUtils.invert(expTs),
+ Bytes.toLong(karr[1]));
+ // key must be empty
+ assertEquals(0, karr[2].length);
+ Object value = e.getValue();
+ // value should be empty
+ assertEquals("", value.toString());
+ }
+ }
+ }
+ assertEquals(1, rowCount);
+
+ // read the timeline entity using the reader this time
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
+
+ // check the events
+ NavigableSet<TimelineEvent> events = e1.getEvents();
+ // there should be only one event
+ assertEquals(1, events.size());
+ for (TimelineEvent e : events) {
+ assertEquals(eventId, e.getId());
+ assertEquals(expTs, e.getTimestamp());
+ Map<String,Object> info = e.getInfo();
+ assertTrue(info == null || info.isEmpty());
+ }
+ } finally {
+ hbi.stop();
+ hbi.close();
+ hbr.stop();;
+ hbr.close();
+ }
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ util.shutdownMiniCluster();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a81fa64a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
deleted file mode 100644
index 2875e01..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ /dev/null
@@ -1,770 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.NavigableSet;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
-import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Various tests to test writing entities to HBase and reading them back from
- * it.
- *
- * It uses a single HBase mini-cluster for all tests which is a little more
- * realistic, and helps test correctness in the presence of other data.
- *
- * Each test uses a different cluster name to be able to handle its own data
- * even if other records exist in the table. Use a different cluster name if
- * you add a new test.
- */
-public class TestHBaseTimelineStorage {
-
- private static HBaseTestingUtility util;
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- util = new HBaseTestingUtility();
- util.startMiniCluster();
- createSchema();
- }
-
- private static void createSchema() throws IOException {
- new EntityTable()
- .createTable(util.getHBaseAdmin(), util.getConfiguration());
- new AppToFlowTable()
- .createTable(util.getHBaseAdmin(), util.getConfiguration());
- new ApplicationTable()
- .createTable(util.getHBaseAdmin(), util.getConfiguration());
- }
-
- @Test
- public void testWriteApplicationToHBase() throws Exception {
- TimelineEntities te = new TimelineEntities();
- ApplicationEntity entity = new ApplicationEntity();
- String id = "hello";
- entity.setId(id);
- long cTime = 1425016501000L;
- long mTime = 1425026901000L;
- entity.setCreatedTime(cTime);
- entity.setModifiedTime(mTime);
-
- // add the info map in Timeline Entity
- Map<String, Object> infoMap = new HashMap<String, Object>();
- infoMap.put("infoMapKey1", "infoMapValue1");
- infoMap.put("infoMapKey2", 10);
- entity.addInfo(infoMap);
-
- // add the isRelatedToEntity info
- String key = "task";
- String value = "is_related_to_entity_id_here";
- Set<String> isRelatedToSet = new HashSet<String>();
- isRelatedToSet.add(value);
- Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
- isRelatedTo.put(key, isRelatedToSet);
- entity.setIsRelatedToEntities(isRelatedTo);
-
- // add the relatesTo info
- key = "container";
- value = "relates_to_entity_id_here";
- Set<String> relatesToSet = new HashSet<String>();
- relatesToSet.add(value);
- value = "relates_to_entity_id_here_Second";
- relatesToSet.add(value);
- Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
- relatesTo.put(key, relatesToSet);
- entity.setRelatesToEntities(relatesTo);
-
- // add some config entries
- Map<String, String> conf = new HashMap<String, String>();
- conf.put("config_param1", "value1");
- conf.put("config_param2", "value2");
- entity.addConfigs(conf);
-
- // add metrics
- Set<TimelineMetric> metrics = new HashSet<>();
- TimelineMetric m1 = new TimelineMetric();
- m1.setId("MAP_SLOT_MILLIS");
- Map<Long, Number> metricValues = new HashMap<Long, Number>();
- long ts = System.currentTimeMillis();
- metricValues.put(ts - 120000, 100000000);
- metricValues.put(ts - 100000, 200000000);
- metricValues.put(ts - 80000, 300000000);
- metricValues.put(ts - 60000, 400000000);
- metricValues.put(ts - 40000, 50000000000L);
- metricValues.put(ts - 20000, 60000000000L);
- m1.setType(Type.TIME_SERIES);
- m1.setValues(metricValues);
- metrics.add(m1);
- entity.addMetrics(metrics);
-
- te.addEntity(entity);
-
- HBaseTimelineWriterImpl hbi = null;
- HBaseTimelineReaderImpl hbr = null;
- try {
- Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
- hbi.init(c1);
- hbi.start();
- hbr = new HBaseTimelineReaderImpl();
- hbr.init(c1);
- hbr.start();
- String cluster = "cluster_test_write_app";
- String user = "user1";
- String flow = "some_flow_name";
- String flowVersion = "AB7822C10F1111";
- long runid = 1002345678919L;
- hbi.write(cluster, user, flow, flowVersion, runid, id, te);
- hbi.stop();
-
- // retrieve the row
- byte[] rowKey =
- ApplicationRowKey.getRowKey(cluster, user, flow, runid, id);
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- Connection conn = ConnectionFactory.createConnection(c1);
- Result result = new ApplicationTable().getResult(c1, conn, get);
-
- assertTrue(result != null);
- assertEquals(16, result.size());
-
- // check the row key
- byte[] row1 = result.getRow();
- assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
- id));
-
- // check info column family
- String id1 = ApplicationColumn.ID.readResult(result).toString();
- assertEquals(id, id1);
-
- Number val =
- (Number) ApplicationColumn.CREATED_TIME.readResult(result);
- long cTime1 = val.longValue();
- assertEquals(cTime1, cTime);
-
- val = (Number) ApplicationColumn.MODIFIED_TIME.readResult(result);
- long mTime1 = val.longValue();
- assertEquals(mTime1, mTime);
-
- Map<String, Object> infoColumns =
- ApplicationColumnPrefix.INFO.readResults(result);
- assertEquals(infoMap, infoColumns);
-
- // Remember isRelatedTo is of type Map<String, Set<String>>
- for (String isRelatedToKey : isRelatedTo.keySet()) {
- Object isRelatedToValue =
- ApplicationColumnPrefix.IS_RELATED_TO.readResult(result,
- isRelatedToKey);
- String compoundValue = isRelatedToValue.toString();
- // id7?id9?id6
- Set<String> isRelatedToValues =
- new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
- assertEquals(isRelatedTo.get(isRelatedToKey).size(),
- isRelatedToValues.size());
- for (String v : isRelatedTo.get(isRelatedToKey)) {
- assertTrue(isRelatedToValues.contains(v));
- }
- }
-
- // RelatesTo
- for (String relatesToKey : relatesTo.keySet()) {
- String compoundValue =
- ApplicationColumnPrefix.RELATES_TO.readResult(result,
- relatesToKey).toString();
- // id3?id4?id5
- Set<String> relatesToValues =
- new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
- assertEquals(relatesTo.get(relatesToKey).size(),
- relatesToValues.size());
- for (String v : relatesTo.get(relatesToKey)) {
- assertTrue(relatesToValues.contains(v));
- }
- }
-
- // Configuration
- Map<String, Object> configColumns =
- ApplicationColumnPrefix.CONFIG.readResults(result);
- assertEquals(conf, configColumns);
-
- NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
-
- NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
- assertEquals(metricValues, metricMap);
-
- // read the timeline entity using the reader this time
- TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, id,
- entity.getType(), entity.getId(),
- EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- id, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- assertNotNull(e1);
- assertEquals(1, es1.size());
-
- // verify attributes
- assertEquals(id, e1.getId());
- assertEquals(TimelineEntityType.YARN_APPLICATION.toString(),
- e1.getType());
- assertEquals(cTime, e1.getCreatedTime());
- assertEquals(mTime, e1.getModifiedTime());
- Map<String, Object> infoMap2 = e1.getInfo();
- assertEquals(infoMap, infoMap2);
-
- Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
- assertEquals(isRelatedTo, isRelatedTo2);
-
- Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
- assertEquals(relatesTo, relatesTo2);
-
- Map<String, String> conf2 = e1.getConfigs();
- assertEquals(conf, conf2);
-
- Set<TimelineMetric> metrics2 = e1.getMetrics();
- assertEquals(metrics, metrics2);
- for (TimelineMetric metric2 : metrics2) {
- Map<Long, Number> metricValues2 = metric2.getValues();
- assertEquals(metricValues, metricValues2);
- }
- } finally {
- if (hbi != null) {
- hbi.stop();
- hbi.close();
- }
- if (hbr != null) {
- hbr.stop();
- hbr.close();
- }
- }
- }
-
- @Test
- public void testWriteEntityToHBase() throws Exception {
- TimelineEntities te = new TimelineEntities();
- TimelineEntity entity = new TimelineEntity();
- String id = "hello";
- String type = "world";
- entity.setId(id);
- entity.setType(type);
- long cTime = 1425016501000L;
- long mTime = 1425026901000L;
- entity.setCreatedTime(cTime);
- entity.setModifiedTime(mTime);
-
- // add the info map in Timeline Entity
- Map<String, Object> infoMap = new HashMap<String, Object>();
- infoMap.put("infoMapKey1", "infoMapValue1");
- infoMap.put("infoMapKey2", 10);
- entity.addInfo(infoMap);
-
- // add the isRelatedToEntity info
- String key = "task";
- String value = "is_related_to_entity_id_here";
- Set<String> isRelatedToSet = new HashSet<String>();
- isRelatedToSet.add(value);
- Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
- isRelatedTo.put(key, isRelatedToSet);
- entity.setIsRelatedToEntities(isRelatedTo);
-
- // add the relatesTo info
- key = "container";
- value = "relates_to_entity_id_here";
- Set<String> relatesToSet = new HashSet<String>();
- relatesToSet.add(value);
- value = "relates_to_entity_id_here_Second";
- relatesToSet.add(value);
- Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
- relatesTo.put(key, relatesToSet);
- entity.setRelatesToEntities(relatesTo);
-
- // add some config entries
- Map<String, String> conf = new HashMap<String, String>();
- conf.put("config_param1", "value1");
- conf.put("config_param2", "value2");
- entity.addConfigs(conf);
-
- // add metrics
- Set<TimelineMetric> metrics = new HashSet<>();
- TimelineMetric m1 = new TimelineMetric();
- m1.setId("MAP_SLOT_MILLIS");
- Map<Long, Number> metricValues = new HashMap<Long, Number>();
- long ts = System.currentTimeMillis();
- metricValues.put(ts - 120000, 100000000);
- metricValues.put(ts - 100000, 200000000);
- metricValues.put(ts - 80000, 300000000);
- metricValues.put(ts - 60000, 400000000);
- metricValues.put(ts - 40000, 50000000000L);
- metricValues.put(ts - 20000, 60000000000L);
- m1.setType(Type.TIME_SERIES);
- m1.setValues(metricValues);
- metrics.add(m1);
- entity.addMetrics(metrics);
-
- te.addEntity(entity);
-
- HBaseTimelineWriterImpl hbi = null;
- HBaseTimelineReaderImpl hbr = null;
- try {
- Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
- hbi.init(c1);
- hbi.start();
- hbr = new HBaseTimelineReaderImpl();
- hbr.init(c1);
- hbr.start();
- String cluster = "cluster_test_write_entity";
- String user = "user1";
- String flow = "some_flow_name";
- String flowVersion = "AB7822C10F1111";
- long runid = 1002345678919L;
- String appName = "some app name";
- hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
- hbi.stop();
-
- // scan the table and see that entity exists
- Scan s = new Scan();
- byte[] startRow =
- EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
- s.setStartRow(startRow);
- s.setMaxVersions(Integer.MAX_VALUE);
- Connection conn = ConnectionFactory.createConnection(c1);
- ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
-
- int rowCount = 0;
- int colCount = 0;
- for (Result result : scanner) {
- if (result != null && !result.isEmpty()) {
- rowCount++;
- colCount += result.size();
- byte[] row1 = result.getRow();
- assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
- entity));
-
- // check info column family
- String id1 = EntityColumn.ID.readResult(result).toString();
- assertEquals(id, id1);
-
- String type1 = EntityColumn.TYPE.readResult(result).toString();
- assertEquals(type, type1);
-
- Number val = (Number) EntityColumn.CREATED_TIME.readResult(result);
- long cTime1 = val.longValue();
- assertEquals(cTime1, cTime);
-
- val = (Number) EntityColumn.MODIFIED_TIME.readResult(result);
- long mTime1 = val.longValue();
- assertEquals(mTime1, mTime);
-
- Map<String, Object> infoColumns =
- EntityColumnPrefix.INFO.readResults(result);
- assertEquals(infoMap, infoColumns);
-
- // Remember isRelatedTo is of type Map<String, Set<String>>
- for (String isRelatedToKey : isRelatedTo.keySet()) {
- Object isRelatedToValue =
- EntityColumnPrefix.IS_RELATED_TO.readResult(result,
- isRelatedToKey);
- String compoundValue = isRelatedToValue.toString();
- // id7?id9?id6
- Set<String> isRelatedToValues =
- new HashSet<String>(
- Separator.VALUES.splitEncoded(compoundValue));
- assertEquals(isRelatedTo.get(isRelatedToKey).size(),
- isRelatedToValues.size());
- for (String v : isRelatedTo.get(isRelatedToKey)) {
- assertTrue(isRelatedToValues.contains(v));
- }
- }
-
- // RelatesTo
- for (String relatesToKey : relatesTo.keySet()) {
- String compoundValue =
- EntityColumnPrefix.RELATES_TO.readResult(result, relatesToKey)
- .toString();
- // id3?id4?id5
- Set<String> relatesToValues =
- new HashSet<String>(
- Separator.VALUES.splitEncoded(compoundValue));
- assertEquals(relatesTo.get(relatesToKey).size(),
- relatesToValues.size());
- for (String v : relatesTo.get(relatesToKey)) {
- assertTrue(relatesToValues.contains(v));
- }
- }
-
- // Configuration
- Map<String, Object> configColumns =
- EntityColumnPrefix.CONFIG.readResults(result);
- assertEquals(conf, configColumns);
-
- NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
-
- NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
- assertEquals(metricValues, metricMap);
- }
- }
- assertEquals(1, rowCount);
- assertEquals(17, colCount);
-
- // read the timeline entity using the reader this time
- TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(),
- EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- assertNotNull(e1);
- assertEquals(1, es1.size());
-
- // verify attributes
- assertEquals(id, e1.getId());
- assertEquals(type, e1.getType());
- assertEquals(cTime, e1.getCreatedTime());
- assertEquals(mTime, e1.getModifiedTime());
- Map<String, Object> infoMap2 = e1.getInfo();
- assertEquals(infoMap, infoMap2);
-
- Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
- assertEquals(isRelatedTo, isRelatedTo2);
-
- Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
- assertEquals(relatesTo, relatesTo2);
-
- Map<String, String> conf2 = e1.getConfigs();
- assertEquals(conf, conf2);
-
- Set<TimelineMetric> metrics2 = e1.getMetrics();
- assertEquals(metrics, metrics2);
- for (TimelineMetric metric2 : metrics2) {
- Map<Long, Number> metricValues2 = metric2.getValues();
- assertEquals(metricValues, metricValues2);
- }
- } finally {
- if (hbi != null) {
- hbi.stop();
- hbi.close();
- }
- if (hbr != null) {
- hbr.stop();
- hbr.close();
- }
- }
- }
-
- private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
- String flow, long runid, String appName, TimelineEntity te) {
-
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
-
- assertTrue(rowKeyComponents.length == 7);
- assertEquals(user, Bytes.toString(rowKeyComponents[0]));
- assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
- assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.invert(runid),
- Bytes.toLong(rowKeyComponents[3]));
- assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
- assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
- assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
- return true;
- }
-
- private boolean isApplicationRowKeyCorrect(byte[] rowKey, String cluster,
- String user, String flow, long runid, String appName) {
-
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
-
- assertTrue(rowKeyComponents.length == 5);
- assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
- assertEquals(user, Bytes.toString(rowKeyComponents[1]));
- assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.invert(runid),
- Bytes.toLong(rowKeyComponents[3]));
- assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
- return true;
- }
-
- @Test
- public void testEvents() throws IOException {
- TimelineEvent event = new TimelineEvent();
- String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
- event.setId(eventId);
- long expTs = 1436512802000L;
- event.setTimestamp(expTs);
- String expKey = "foo_event";
- Object expVal = "test";
- event.addInfo(expKey, expVal);
-
- final TimelineEntity entity = new ApplicationEntity();
- entity.setId(ApplicationId.newInstance(0, 1).toString());
- entity.addEvent(event);
-
- TimelineEntities entities = new TimelineEntities();
- entities.addEntity(entity);
-
- HBaseTimelineWriterImpl hbi = null;
- HBaseTimelineReaderImpl hbr = null;
- try {
- Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
- hbi.init(c1);
- hbi.start();
- hbr = new HBaseTimelineReaderImpl();
- hbr.init(c1);
- hbr.start();
- String cluster = "cluster_test_events";
- String user = "user2";
- String flow = "other_flow_name";
- String flowVersion = "1111F01C2287BA";
- long runid = 1009876543218L;
- String appName = "some app name";
- hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
- hbi.stop();
-
- // retrieve the row
- byte[] rowKey =
- ApplicationRowKey.getRowKey(cluster, user, flow, runid, appName);
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- Connection conn = ConnectionFactory.createConnection(c1);
- Result result = new ApplicationTable().getResult(c1, conn, get);
-
- assertTrue(result != null);
-
- // check the row key
- byte[] row1 = result.getRow();
- assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
- appName));
-
- Map<?, Object> eventsResult =
- ApplicationColumnPrefix.EVENT.
- readResultsHavingCompoundColumnQualifiers(result);
- // there should be only one event
- assertEquals(1, eventsResult.size());
- for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
- // the qualifier is a compound key
- // hence match individual values
- byte[][] karr = (byte[][])e.getKey();
- assertEquals(3, karr.length);
- assertEquals(eventId, Bytes.toString(karr[0]));
- assertEquals(TimelineWriterUtils.invert(expTs), Bytes.toLong(karr[1]));
- assertEquals(expKey, Bytes.toString(karr[2]));
- Object value = e.getValue();
- // there should be only one timestamp and value
- assertEquals(expVal, value.toString());
- }
-
- // read the timeline entity using the reader this time
- TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(),
- EnumSet.of(TimelineReader.Field.ALL));
- TimelineEntity e2 = hbr.getEntity(user, cluster, null, null, appName,
- entity.getType(), entity.getId(),
- EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es2 = hbr.getEntities(user, cluster, null, null,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- assertNotNull(e1);
- assertNotNull(e2);
- assertEquals(e1, e2);
- assertEquals(1, es1.size());
- assertEquals(1, es2.size());
- assertEquals(es1, es2);
-
- // check the events
- NavigableSet<TimelineEvent> events = e1.getEvents();
- // there should be only one event
- assertEquals(1, events.size());
- for (TimelineEvent e : events) {
- assertEquals(eventId, e.getId());
- assertEquals(expTs, e.getTimestamp());
- Map<String,Object> info = e.getInfo();
- assertEquals(1, info.size());
- for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
- assertEquals(expKey, infoEntry.getKey());
- assertEquals(expVal, infoEntry.getValue());
- }
- }
- } finally {
- if (hbi != null) {
- hbi.stop();
- hbi.close();
- }
- if (hbr != null) {
- hbr.stop();
- hbr.close();
- }
- }
- }
-
- @Test
- public void testEventsWithEmptyInfo() throws IOException {
- TimelineEvent event = new TimelineEvent();
- String eventId = "foo_event_id";
- event.setId(eventId);
- long expTs = 1436512802000L;
- event.setTimestamp(expTs);
-
- final TimelineEntity entity = new TimelineEntity();
- entity.setId("attempt_1329348432655_0001_m_000008_18");
- entity.setType("FOO_ATTEMPT");
- entity.addEvent(event);
-
- TimelineEntities entities = new TimelineEntities();
- entities.addEntity(entity);
-
- HBaseTimelineWriterImpl hbi = null;
- HBaseTimelineReaderImpl hbr = null;
- try {
- Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
- hbi.init(c1);
- hbi.start();
- hbr = new HBaseTimelineReaderImpl();
- hbr.init(c1);
- hbr.start();
- String cluster = "cluster_test_empty_eventkey";
- String user = "user_emptyeventkey";
- String flow = "other_flow_name";
- String flowVersion = "1111F01C2287BA";
- long runid = 1009876543218L;
- String appName = "some app name";
- byte[] startRow =
- EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
- hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
- hbi.stop();
- // scan the table and see that entity exists
- Scan s = new Scan();
- s.setStartRow(startRow);
- s.addFamily(EntityColumnFamily.INFO.getBytes());
- Connection conn = ConnectionFactory.createConnection(c1);
- ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
-
- int rowCount = 0;
- for (Result result : scanner) {
- if (result != null && !result.isEmpty()) {
- rowCount++;
-
- // check the row key
- byte[] row1 = result.getRow();
- assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
- entity));
-
- Map<?, Object> eventsResult =
- EntityColumnPrefix.EVENT.
- readResultsHavingCompoundColumnQualifiers(result);
- // there should be only one event
- assertEquals(1, eventsResult.size());
- for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
- // the qualifier is a compound key
- // hence match individual values
- byte[][] karr = (byte[][])e.getKey();
- assertEquals(3, karr.length);
- assertEquals(eventId, Bytes.toString(karr[0]));
- assertEquals(TimelineWriterUtils.invert(expTs),
- Bytes.toLong(karr[1]));
- // key must be empty
- assertEquals(0, karr[2].length);
- Object value = e.getValue();
- // value should be empty
- assertEquals("", value.toString());
- }
- }
- }
- assertEquals(1, rowCount);
-
- // read the timeline entity using the reader this time
- TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(),
- EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- assertNotNull(e1);
- assertEquals(1, es1.size());
-
- // check the events
- NavigableSet<TimelineEvent> events = e1.getEvents();
- // there should be only one event
- assertEquals(1, events.size());
- for (TimelineEvent e : events) {
- assertEquals(eventId, e.getId());
- assertEquals(expTs, e.getTimestamp());
- Map<String,Object> info = e.getInfo();
- assertTrue(info == null || info.isEmpty());
- }
- } finally {
- hbi.stop();
- hbi.close();
- hbr.stop();;
- hbr.close();
- }
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- util.shutdownMiniCluster();
- }
-}
[28/50] [abbrv] hadoop git commit: YARN-3906. Split the application
table from the entity table. Contributed by Sangjin Lee.
Posted by sj...@apache.org.
YARN-3906. Split the application table from the entity table. Contributed by Sangjin Lee.
(cherry picked from commit bcd755eba9466ce277d3c14192c31da6462c4ab3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/491b3023
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/491b3023
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/491b3023
Branch: refs/heads/YARN-2928-rebase
Commit: 491b3023cac8c6ae39d39d677dc1b41ae5a16cc9
Parents: 2ecbdad
Author: Junping Du <ju...@apache.org>
Authored: Tue Aug 11 16:59:21 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:41 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../storage/HBaseTimelineReaderImpl.java | 202 ++++++++----
.../storage/HBaseTimelineWriterImpl.java | 145 ++++++---
.../storage/TimelineSchemaCreator.java | 10 +
.../storage/application/ApplicationColumn.java | 136 ++++++++
.../application/ApplicationColumnFamily.java | 65 ++++
.../application/ApplicationColumnPrefix.java | 217 +++++++++++++
.../storage/application/ApplicationRowKey.java | 67 ++++
.../storage/application/ApplicationTable.java | 164 ++++++++++
.../storage/application/package-info.java | 25 ++
.../storage/entity/EntityColumnPrefix.java | 2 +-
.../storage/entity/EntityTable.java | 59 ++--
.../storage/TestHBaseTimelineWriterImpl.java | 321 ++++++++++++++++---
13 files changed, 1230 insertions(+), 186 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 922d54a..d4e2c4f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -85,6 +85,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3049. [Storage Implementation] Implement storage reader interface to
fetch raw data from HBase backend (Zhijie Shen via sjlee)
+ YARN-3906. Split the application table from the entity table. (Sangjin Lee
+ via junping_du)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index 5258b9c..094f868 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -18,7 +18,19 @@
package org.apache.hadoop.yarn.server.timelineservice.storage;
-import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -32,11 +44,17 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
@@ -45,18 +63,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.TreeSet;
+import com.google.common.base.Preconditions;
public class HBaseTimelineReaderImpl
extends AbstractService implements TimelineReader {
@@ -70,6 +77,7 @@ public class HBaseTimelineReaderImpl
private Connection conn;
private EntityTable entityTable;
private AppToFlowTable appToFlowTable;
+ private ApplicationTable applicationTable;
public HBaseTimelineReaderImpl() {
super(HBaseTimelineReaderImpl.class.getName());
@@ -82,6 +90,7 @@ public class HBaseTimelineReaderImpl
conn = ConnectionFactory.createConnection(hbaseConf);
entityTable = new EntityTable();
appToFlowTable = new AppToFlowTable();
+ applicationTable = new ApplicationTable();
}
@Override
@@ -109,14 +118,24 @@ public class HBaseTimelineReaderImpl
fieldsToRetrieve = EnumSet.noneOf(Field.class);
}
- byte[] rowKey = EntityRowKey.getRowKey(
- clusterId, userId, flowId, flowRunId, appId, entityType, entityId);
+ boolean isApplication = isApplicationEntity(entityType);
+ byte[] rowKey = isApplication ?
+ ApplicationRowKey.getRowKey(clusterId, userId, flowId, flowRunId,
+ appId) :
+ EntityRowKey.getRowKey(clusterId, userId, flowId, flowRunId, appId,
+ entityType, entityId);
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
- return parseEntity(
- entityTable.getResult(hbaseConf, conn, get), fieldsToRetrieve,
+ Result result = isApplication ?
+ applicationTable.getResult(hbaseConf, conn, get) :
+ entityTable.getResult(hbaseConf, conn, get);
+ return parseEntity(result, fieldsToRetrieve,
false, DEFAULT_BEGIN_TIME, DEFAULT_END_TIME, false, DEFAULT_BEGIN_TIME,
- DEFAULT_END_TIME, null, null, null, null, null, null);
+ DEFAULT_END_TIME, null, null, null, null, null, null, isApplication);
+ }
+
+ private static boolean isApplicationEntity(String entityType) {
+ return TimelineEntityType.YARN_APPLICATION.toString().equals(entityType);
}
@Override
@@ -155,26 +174,46 @@ public class HBaseTimelineReaderImpl
}
NavigableSet<TimelineEntity> entities = new TreeSet<>();
- // Scan through part of the table to find the entities belong to one app and
- // one type
- Scan scan = new Scan();
- scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
- clusterId, userId, flowId, flowRunId, appId, entityType));
- scan.setMaxVersions(Integer.MAX_VALUE);
- ResultScanner scanner = entityTable.getResultScanner(hbaseConf, conn, scan);
- for (Result result : scanner) {
+ boolean isApplication = isApplicationEntity(entityType);
+ if (isApplication) {
+ // If getEntities() is called for an application, there can be at most
+ // one entity. If the entity passes the filter, it is returned. Otherwise,
+ // an empty set is returned.
+ byte[] rowKey = ApplicationRowKey.getRowKey(clusterId, userId, flowId,
+ flowRunId, appId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Result result = applicationTable.getResult(hbaseConf, conn, get);
TimelineEntity entity = parseEntity(result, fieldsToRetrieve,
- true, createdTimeBegin, createdTimeEnd,
- true, modifiedTimeBegin, modifiedTimeEnd,
- isRelatedTo, relatesTo, infoFilters, configFilters, eventFilters,
- metricFilters);
- if (entity == null) {
- continue;
+ true, createdTimeBegin, createdTimeEnd, true, modifiedTimeBegin,
+ modifiedTimeEnd, isRelatedTo, relatesTo, infoFilters, configFilters,
+ eventFilters, metricFilters, isApplication);
+ if (entity != null) {
+ entities.add(entity);
}
- if (entities.size() > limit) {
- entities.pollLast();
+ } else {
+ // Scan through part of the table to find the entities belong to one app
+ // and one type
+ Scan scan = new Scan();
+ scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
+ clusterId, userId, flowId, flowRunId, appId, entityType));
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ ResultScanner scanner =
+ entityTable.getResultScanner(hbaseConf, conn, scan);
+ for (Result result : scanner) {
+ TimelineEntity entity = parseEntity(result, fieldsToRetrieve,
+ true, createdTimeBegin, createdTimeEnd,
+ true, modifiedTimeBegin, modifiedTimeEnd,
+ isRelatedTo, relatesTo, infoFilters, configFilters, eventFilters,
+ metricFilters, isApplication);
+ if (entity == null) {
+ continue;
+ }
+ if (entities.size() > limit) {
+ entities.pollLast();
+ }
+ entities.add(entity);
}
- entities.add(entity);
}
return entities;
}
@@ -221,26 +260,37 @@ public class HBaseTimelineReaderImpl
boolean checkModifiedTime, long modifiedTimeBegin, long modifiedTimeEnd,
Map<String, Set<String>> isRelatedTo, Map<String, Set<String>> relatesTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> eventFilters, Set<String> metricFilters)
+ Set<String> eventFilters, Set<String> metricFilters,
+ boolean isApplication)
throws IOException {
if (result == null || result.isEmpty()) {
return null;
}
TimelineEntity entity = new TimelineEntity();
- entity.setType(EntityColumn.TYPE.readResult(result).toString());
- entity.setId(EntityColumn.ID.readResult(result).toString());
+ String entityType = isApplication ?
+ TimelineEntityType.YARN_APPLICATION.toString() :
+ EntityColumn.TYPE.readResult(result).toString();
+ entity.setType(entityType);
+ String entityId = isApplication ?
+ ApplicationColumn.ID.readResult(result).toString() :
+ EntityColumn.ID.readResult(result).toString();
+ entity.setId(entityId);
// fetch created time
- entity.setCreatedTime(
- ((Number) EntityColumn.CREATED_TIME.readResult(result)).longValue());
+ Number createdTime = isApplication ?
+ (Number)ApplicationColumn.CREATED_TIME.readResult(result) :
+ (Number)EntityColumn.CREATED_TIME.readResult(result);
+ entity.setCreatedTime(createdTime.longValue());
if (checkCreatedTime && (entity.getCreatedTime() < createdTimeBegin ||
entity.getCreatedTime() > createdTimeEnd)) {
return null;
}
// fetch modified time
- entity.setCreatedTime(
- ((Number) EntityColumn.MODIFIED_TIME.readResult(result)).longValue());
+ Number modifiedTime = isApplication ?
+ (Number)ApplicationColumn.MODIFIED_TIME.readResult(result) :
+ (Number)EntityColumn.MODIFIED_TIME.readResult(result);
+ entity.setModifiedTime(modifiedTime.longValue());
if (checkModifiedTime && (entity.getModifiedTime() < modifiedTimeBegin ||
entity.getModifiedTime() > modifiedTimeEnd)) {
return null;
@@ -250,7 +300,13 @@ public class HBaseTimelineReaderImpl
boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
- readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO);
+ if (isApplication) {
+ readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
+ true);
+ } else {
+ readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO,
+ true);
+ }
if (checkIsRelatedTo && !TimelineReaderUtils.matchRelations(
entity.getIsRelatedToEntities(), isRelatedTo)) {
return null;
@@ -265,7 +321,12 @@ public class HBaseTimelineReaderImpl
boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
- readRelationship(entity, result, EntityColumnPrefix.RELATES_TO);
+ if (isApplication) {
+ readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
+ false);
+ } else {
+ readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
+ }
if (checkRelatesTo && !TimelineReaderUtils.matchRelations(
entity.getRelatesToEntities(), relatesTo)) {
return null;
@@ -280,7 +341,11 @@ public class HBaseTimelineReaderImpl
boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
- readKeyValuePairs(entity, result, EntityColumnPrefix.INFO);
+ if (isApplication) {
+ readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
+ } else {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
+ }
if (checkInfo &&
!TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
return null;
@@ -295,7 +360,11 @@ public class HBaseTimelineReaderImpl
boolean checkConfigs = configFilters != null && configFilters.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
- readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG);
+ if (isApplication) {
+ readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
+ } else {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
+ }
if (checkConfigs && !TimelineReaderUtils.matchFilters(
entity.getConfigs(), configFilters)) {
return null;
@@ -310,7 +379,7 @@ public class HBaseTimelineReaderImpl
boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
- readEvents(entity, result);
+ readEvents(entity, result, isApplication);
if (checkEvents && !TimelineReaderUtils.matchEventFilters(
entity.getEvents(), eventFilters)) {
return null;
@@ -325,7 +394,7 @@ public class HBaseTimelineReaderImpl
boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
- readMetrics(entity, result);
+ readMetrics(entity, result, isApplication);
if (checkMetrics && !TimelineReaderUtils.matchMetricFilters(
entity.getMetrics(), metricFilters)) {
return null;
@@ -338,15 +407,15 @@ public class HBaseTimelineReaderImpl
return entity;
}
- private static void readRelationship(
- TimelineEntity entity, Result result, EntityColumnPrefix prefix)
- throws IOException {
+ private static <T> void readRelationship(
+ TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
+ boolean isRelatedTo) throws IOException {
// isRelatedTo and relatesTo are of type Map<String, Set<String>>
Map<String, Object> columns = prefix.readResults(result);
for (Map.Entry<String, Object> column : columns.entrySet()) {
for (String id : Separator.VALUES.splitEncoded(
column.getValue().toString())) {
- if (prefix.equals(EntityColumnPrefix.IS_RELATED_TO)) {
+ if (isRelatedTo) {
entity.addIsRelatedToEntity(column.getKey(), id);
} else {
entity.addRelatesToEntity(column.getKey(), id);
@@ -355,12 +424,12 @@ public class HBaseTimelineReaderImpl
}
}
- private static void readKeyValuePairs(
- TimelineEntity entity, Result result, EntityColumnPrefix prefix)
- throws IOException {
+ private static <T> void readKeyValuePairs(
+ TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
+ boolean isConfig) throws IOException {
// info and configuration are of type Map<String, Object or String>
Map<String, Object> columns = prefix.readResults(result);
- if (prefix.equals(EntityColumnPrefix.CONFIG)) {
+ if (isConfig) {
for (Map.Entry<String, Object> column : columns.entrySet()) {
entity.addConfig(column.getKey(), column.getKey().toString());
}
@@ -369,10 +438,11 @@ public class HBaseTimelineReaderImpl
}
}
- private static void readEvents(TimelineEntity entity, Result result)
- throws IOException {
+ private static void readEvents(TimelineEntity entity, Result result,
+ boolean isApplication) throws IOException {
Map<String, TimelineEvent> eventsMap = new HashMap<>();
- Map<String, Object> eventsResult =
+ Map<String, Object> eventsResult = isApplication ?
+ ApplicationColumnPrefix.EVENT.readResults(result) :
EntityColumnPrefix.EVENT.readResults(result);
for (Map.Entry<String,Object> eventResult : eventsResult.entrySet()) {
Collection<String> tokens =
@@ -405,10 +475,16 @@ public class HBaseTimelineReaderImpl
entity.addEvents(eventsSet);
}
- private static void readMetrics(TimelineEntity entity, Result result)
- throws IOException {
- NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
+ private static void readMetrics(TimelineEntity entity, Result result,
+ boolean isApplication) throws IOException {
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult;
+ if (isApplication) {
+ metricsResult =
+ ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
+ } else {
+ metricsResult =
+ EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
+ }
for (Map.Entry<String, NavigableMap<Long, Number>> metricResult:
metricsResult.entrySet()) {
TimelineMetric metric = new TimelineMetric();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 5290415..96192cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -38,9 +38,14 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
@@ -61,6 +66,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
private Connection conn;
private TypedBufferedMutator<EntityTable> entityTable;
private TypedBufferedMutator<AppToFlowTable> appToFlowTable;
+ private TypedBufferedMutator<ApplicationTable> applicationTable;
private static final Log LOG = LogFactory
.getLog(HBaseTimelineWriterImpl.class);
@@ -84,6 +90,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
conn = ConnectionFactory.createConnection(hbaseConf);
entityTable = new EntityTable().getTableMutator(hbaseConf, conn);
appToFlowTable = new AppToFlowTable().getTableMutator(hbaseConf, conn);
+ applicationTable = new ApplicationTable().getTableMutator(hbaseConf, conn);
}
/**
@@ -102,18 +109,20 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
continue;
}
- byte[] rowKey =
+ // if the entity is the application, the destination is the application
+ // table
+ boolean isApplication = isApplicationEntity(te);
+ byte[] rowKey = isApplication ?
+ ApplicationRowKey.getRowKey(clusterId, userId, flowName, flowRunId,
+ appId) :
EntityRowKey.getRowKey(clusterId, userId, flowName, flowRunId, appId,
te.getType(), te.getId());
- storeInfo(rowKey, te, flowVersion);
- storeEvents(rowKey, te.getEvents());
- storeConfig(rowKey, te.getConfigs());
- storeMetrics(rowKey, te.getMetrics());
- storeRelations(rowKey, te.getIsRelatedToEntities(),
- EntityColumnPrefix.IS_RELATED_TO);
- storeRelations(rowKey, te.getRelatesToEntities(),
- EntityColumnPrefix.RELATES_TO);
+ storeInfo(rowKey, te, flowVersion, isApplication);
+ storeEvents(rowKey, te.getEvents(), isApplication);
+ storeConfig(rowKey, te.getConfigs(), isApplication);
+ storeMetrics(rowKey, te.getMetrics(), isApplication);
+ storeRelations(rowKey, te, isApplication);
if (isApplicationCreated(te)) {
onApplicationCreated(
@@ -123,9 +132,12 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
return putStatus;
}
+ private static boolean isApplicationEntity(TimelineEntity te) {
+ return te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString());
+ }
+
private static boolean isApplicationCreated(TimelineEntity te) {
- if (te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString())) {
- boolean isAppCreated = false;
+ if (isApplicationEntity(te)) {
for (TimelineEvent event : te.getEvents()) {
if (event.getId().equals(
ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
@@ -145,41 +157,74 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
rowKey, appToFlowTable, null, flowRunId);
}
+ private void storeRelations(byte[] rowKey, TimelineEntity te,
+ boolean isApplication) throws IOException {
+ if (isApplication) {
+ storeRelations(rowKey, te.getIsRelatedToEntities(),
+ ApplicationColumnPrefix.IS_RELATED_TO, applicationTable);
+ storeRelations(rowKey, te.getRelatesToEntities(),
+ ApplicationColumnPrefix.RELATES_TO, applicationTable);
+ } else {
+ storeRelations(rowKey, te.getIsRelatedToEntities(),
+ EntityColumnPrefix.IS_RELATED_TO, entityTable);
+ storeRelations(rowKey, te.getRelatesToEntities(),
+ EntityColumnPrefix.RELATES_TO, entityTable);
+ }
+ }
+
/**
* Stores the Relations from the {@linkplain TimelineEntity} object
*/
- private void storeRelations(byte[] rowKey,
+ private <T> void storeRelations(byte[] rowKey,
Map<String, Set<String>> connectedEntities,
- EntityColumnPrefix entityColumnPrefix) throws IOException {
+ ColumnPrefix<T> columnPrefix, TypedBufferedMutator<T> table)
+ throws IOException {
for (Map.Entry<String, Set<String>> connectedEntity : connectedEntities
.entrySet()) {
// id3?id4?id5
String compoundValue =
Separator.VALUES.joinEncoded(connectedEntity.getValue());
- entityColumnPrefix.store(rowKey, entityTable, connectedEntity.getKey(),
- null, compoundValue);
+ columnPrefix.store(rowKey, table, connectedEntity.getKey(), null,
+ compoundValue);
}
}
/**
* Stores information from the {@linkplain TimelineEntity} object
*/
- private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion)
- throws IOException {
+ private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion,
+ boolean isApplication) throws IOException {
- EntityColumn.ID.store(rowKey, entityTable, null, te.getId());
- EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType());
- EntityColumn.CREATED_TIME.store(rowKey, entityTable, null,
- te.getCreatedTime());
- EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null,
- te.getModifiedTime());
- EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
- Map<String, Object> info = te.getInfo();
- if (info != null) {
- for (Map.Entry<String, Object> entry : info.entrySet()) {
- EntityColumnPrefix.INFO.store(rowKey, entityTable, entry.getKey(),
- null, entry.getValue());
+ if (isApplication) {
+ ApplicationColumn.ID.store(rowKey, applicationTable, null, te.getId());
+ ApplicationColumn.CREATED_TIME.store(rowKey, applicationTable, null,
+ te.getCreatedTime());
+ ApplicationColumn.MODIFIED_TIME.store(rowKey, applicationTable, null,
+ te.getModifiedTime());
+ ApplicationColumn.FLOW_VERSION.store(rowKey, applicationTable, null,
+ flowVersion);
+ Map<String, Object> info = te.getInfo();
+ if (info != null) {
+ for (Map.Entry<String, Object> entry : info.entrySet()) {
+ ApplicationColumnPrefix.INFO.store(rowKey, applicationTable,
+ entry.getKey(), null, entry.getValue());
+ }
+ }
+ } else {
+ EntityColumn.ID.store(rowKey, entityTable, null, te.getId());
+ EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType());
+ EntityColumn.CREATED_TIME.store(rowKey, entityTable, null,
+ te.getCreatedTime());
+ EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null,
+ te.getModifiedTime());
+ EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
+ Map<String, Object> info = te.getInfo();
+ if (info != null) {
+ for (Map.Entry<String, Object> entry : info.entrySet()) {
+ EntityColumnPrefix.INFO.store(rowKey, entityTable, entry.getKey(),
+ null, entry.getValue());
+ }
}
}
}
@@ -187,14 +232,19 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
/**
* stores the config information from {@linkplain TimelineEntity}
*/
- private void storeConfig(byte[] rowKey, Map<String, String> config)
- throws IOException {
+ private void storeConfig(byte[] rowKey, Map<String, String> config,
+ boolean isApplication) throws IOException {
if (config == null) {
return;
}
for (Map.Entry<String, String> entry : config.entrySet()) {
- EntityColumnPrefix.CONFIG.store(rowKey, entityTable, entry.getKey(),
+ if (isApplication) {
+ ApplicationColumnPrefix.CONFIG.store(rowKey, applicationTable,
+ entry.getKey(), null, entry.getValue());
+ } else {
+ EntityColumnPrefix.CONFIG.store(rowKey, entityTable, entry.getKey(),
null, entry.getValue());
+ }
}
}
@@ -202,16 +252,21 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
* stores the {@linkplain TimelineMetric} information from the
* {@linkplain TimelineEvent} object
*/
- private void storeMetrics(byte[] rowKey, Set<TimelineMetric> metrics)
- throws IOException {
+ private void storeMetrics(byte[] rowKey, Set<TimelineMetric> metrics,
+ boolean isApplication) throws IOException {
if (metrics != null) {
for (TimelineMetric metric : metrics) {
String metricColumnQualifier = metric.getId();
Map<Long, Number> timeseries = metric.getValues();
for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
Long timestamp = timeseriesEntry.getKey();
- EntityColumnPrefix.METRIC.store(rowKey, entityTable,
+ if (isApplication) {
+ ApplicationColumnPrefix.METRIC.store(rowKey, applicationTable,
metricColumnQualifier, timestamp, timeseriesEntry.getValue());
+ } else {
+ EntityColumnPrefix.METRIC.store(rowKey, entityTable,
+ metricColumnQualifier, timestamp, timeseriesEntry.getValue());
+ }
}
}
}
@@ -220,8 +275,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
/**
* Stores the events from the {@linkplain TimelineEvent} object
*/
- private void storeEvents(byte[] rowKey, Set<TimelineEvent> events)
- throws IOException {
+ private void storeEvents(byte[] rowKey, Set<TimelineEvent> events,
+ boolean isApplication) throws IOException {
if (events != null) {
for (TimelineEvent event : events) {
if (event != null) {
@@ -258,8 +313,13 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// convert back to string to avoid additional API on store.
String compoundColumnQualifier =
Bytes.toString(compoundColumnQualifierBytes);
- EntityColumnPrefix.EVENT.store(rowKey, entityTable,
+ if (isApplication) {
+ ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable,
compoundColumnQualifier, null, info.getValue());
+ } else {
+ EntityColumnPrefix.EVENT.store(rowKey, entityTable,
+ compoundColumnQualifier, null, info.getValue());
+ }
} // for info: eventInfo
}
}
@@ -279,6 +339,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// flush all buffered mutators
entityTable.flush();
appToFlowTable.flush();
+ applicationTable.flush();
}
/**
@@ -288,15 +349,19 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
@Override
protected void serviceStop() throws Exception {
if (entityTable != null) {
- LOG.info("closing entity table");
+ LOG.info("closing the entity table");
// The close API performs flushing and releases any resources held
entityTable.close();
}
if (appToFlowTable != null) {
- LOG.info("closing app_flow table");
+ LOG.info("closing the app_flow table");
// The close API performs flushing and releases any resources held
appToFlowTable.close();
}
+ if (applicationTable != null) {
+ LOG.info("closing the application table");
+ applicationTable.close();
+ }
if (conn != null) {
LOG.info("closing the hbase Connection");
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index 2c3897d..3a22ed6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
@@ -76,6 +77,12 @@ public class TimelineSchemaCreator {
if (StringUtils.isNotBlank(appToflowTableName)) {
hbaseConf.set(AppToFlowTable.TABLE_NAME_CONF_NAME, appToflowTableName);
}
+ // Grab the applicationTableName argument
+ String applicationTableName = commandLine.getOptionValue("a");
+ if (StringUtils.isNotBlank(applicationTableName)) {
+ hbaseConf.set(ApplicationTable.TABLE_NAME_CONF_NAME,
+ applicationTableName);
+ }
createAllTables(hbaseConf);
}
@@ -103,6 +110,8 @@ public class TimelineSchemaCreator {
o = new Option("a2f", "appToflowTableName", true, "app to flow table name");
o.setArgName("appToflowTableName");
+ o = new Option("a", "applicationTableName", true, "application table name");
+ o.setArgName("applicationTableName");
o.setRequired(false);
options.addOption(o);
@@ -132,6 +141,7 @@ public class TimelineSchemaCreator {
}
new EntityTable().createTable(admin, hbaseConf);
new AppToFlowTable().createTable(admin, hbaseConf);
+ new ApplicationTable().createTable(admin, hbaseConf);
} finally {
if (conn != null) {
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
new file mode 100644
index 0000000..c028386
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies fully qualified columns for the {@link ApplicationTable}.
+ */
+public enum ApplicationColumn implements Column<ApplicationTable> {
+
+ /**
+ * App id
+ */
+ ID(ApplicationColumnFamily.INFO, "id"),
+
+ /**
+ * When the application was created.
+ */
+ CREATED_TIME(ApplicationColumnFamily.INFO, "created_time"),
+
+ /**
+ * When it was modified.
+ */
+ MODIFIED_TIME(ApplicationColumnFamily.INFO, "modified_time"),
+
+ /**
+ * The version of the flow that this app belongs to.
+ */
+ FLOW_VERSION(ApplicationColumnFamily.INFO, "flow_version");
+
+ private final ColumnHelper<ApplicationTable> column;
+ private final ColumnFamily<ApplicationTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+
+ private ApplicationColumn(ColumnFamily<ApplicationTable> columnFamily,
+ String columnQualifier) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.column = new ColumnHelper<ApplicationTable>(columnFamily);
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<ApplicationTable> tableMutator, Long timestamp,
+ Object inputValue) throws IOException {
+ column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
+ inputValue);
+ }
+
+ public Object readResult(Result result) throws IOException {
+ return column.readResult(result, columnQualifierBytes);
+ }
+
+ /**
+ * Retrieve an {@link ApplicationColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(x) == columnFor(y)} if
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnQualifier Name of the column to retrieve
+ * @return the corresponding {@link ApplicationColumn} or null
+ */
+ public static final ApplicationColumn columnFor(String columnQualifier) {
+
+ // Match column based on value, assume column family matches.
+ for (ApplicationColumn ac : ApplicationColumn.values()) {
+ // Find a match based only on name.
+ if (ac.getColumnQualifier().equals(columnQualifier)) {
+ return ac;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link ApplicationColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)}
+ * if and only if {@code a.equals(b) & x.equals(y)} or
+ * {@code (x == y == null)}
+ *
+ * @param columnFamily The columnFamily for which to retrieve the column.
+ * @param name Name of the column to retrieve
+ * @return the corresponding {@link ApplicationColumn} or null if both
+ * arguments don't match.
+ */
+ public static final ApplicationColumn columnFor(
+ ApplicationColumnFamily columnFamily, String name) {
+
+ for (ApplicationColumn ac : ApplicationColumn.values()) {
+ // Find a match based column family and on name.
+ if (ac.columnFamily.equals(columnFamily)
+ && ac.getColumnQualifier().equals(name)) {
+ return ac;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
new file mode 100644
index 0000000..97e5f7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the application table column families.
+ */
+public enum ApplicationColumnFamily implements ColumnFamily<ApplicationTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i"),
+
+ /**
+ * Configurations are in a separate column family for two reasons: a) the size
+ * of the config values can be very large and b) we expect that config values
+ * are often separately accessed from other metrics and info columns.
+ */
+ CONFIGS("c"),
+
+ /**
+ * Metrics have a separate column family, because they have a separate TTL.
+ */
+ METRICS("m");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ private ApplicationColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
new file mode 100644
index 0000000..cd9e845
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies partially qualified columns for the application table.
+ */
+public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
+
+ /**
+ * To store TimelineEntity getIsRelatedToEntities values.
+ */
+ IS_RELATED_TO(ApplicationColumnFamily.INFO, "s"),
+
+ /**
+ * To store TimelineEntity getRelatesToEntities values.
+ */
+ RELATES_TO(ApplicationColumnFamily.INFO, "r"),
+
+ /**
+ * To store TimelineEntity info values.
+ */
+ INFO(ApplicationColumnFamily.INFO, "i"),
+
+ /**
+ * Lifecycle events for an application
+ */
+ EVENT(ApplicationColumnFamily.INFO, "e"),
+
+ /**
+ * Config column stores configuration with config key as the column name.
+ */
+ CONFIG(ApplicationColumnFamily.CONFIGS, null),
+
+ /**
+ * Metrics are stored with the metric name as the column name.
+ */
+ METRIC(ApplicationColumnFamily.METRICS, null);
+
+ private final ColumnHelper<ApplicationTable> column;
+ private final ColumnFamily<ApplicationTable> columnFamily;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ */
+ private ApplicationColumnPrefix(ColumnFamily<ApplicationTable> columnFamily,
+ String columnPrefix) {
+ column = new ColumnHelper<ApplicationTable>(columnFamily);
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
+ }
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ */
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<ApplicationTable> tableMutator, String qualifier,
+ Long timestamp, Object inputValue) throws IOException {
+
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
+ */
+ public Object readResult(Result result, String qualifier) throws IOException {
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+ return column.readResult(result, columnQualifier);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResults(org.apache.hadoop.hbase.client.Result)
+ */
+ public Map<String, Object> readResults(Result result) throws IOException {
+ return column.readResults(result, columnPrefixBytes);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
+ */
+ public <V> NavigableMap<String, NavigableMap<Long, V>>
+ readResultsWithTimestamps(Result result) throws IOException {
+ return column.readResultsWithTimestamps(result, columnPrefixBytes);
+ }
+
+ /**
+ * Retrieve an {@link ApplicationColumnPrefix} given a name, or null if there
+ * is no match. The following holds true: {@code columnFor(x) == columnFor(y)}
+ * if and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnPrefix Name of the column to retrieve
+ * @return the corresponding {@link ApplicationColumnPrefix} or null
+ */
+ public static final ApplicationColumnPrefix columnFor(String columnPrefix) {
+
+ // Match column based on value, assume column family matches.
+ for (ApplicationColumnPrefix acp : ApplicationColumnPrefix.values()) {
+ // Find a match based only on name.
+ if (acp.getColumnPrefix().equals(columnPrefix)) {
+ return acp;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link ApplicationColumnPrefix} given a name, or null if there
+ * is no match. The following holds true:
+ * {@code columnFor(a,x) == columnFor(b,y)} if and only if
+ * {@code (x == y == null)} or {@code a.equals(b) & x.equals(y)}
+ *
+ * @param columnFamily The columnFamily for which to retrieve the column.
+ * @param columnPrefix Name of the column to retrieve
+ * @return the corresponding {@link ApplicationColumnPrefix} or null if both
+ * arguments don't match.
+ */
+ public static final ApplicationColumnPrefix columnFor(
+ ApplicationColumnFamily columnFamily, String columnPrefix) {
+
+ // TODO: needs unit test to confirm and need to update javadoc to explain
+ // null prefix case.
+
+ for (ApplicationColumnPrefix acp : ApplicationColumnPrefix.values()) {
+ // Find a match based column family and on name.
+ if (acp.columnFamily.equals(columnFamily)
+ && (((columnPrefix == null) && (acp.getColumnPrefix() == null)) || (acp
+ .getColumnPrefix().equals(columnPrefix)))) {
+ return acp;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
new file mode 100644
index 0000000..5f3868b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the application table.
+ */
+public class ApplicationRowKey {
+ // TODO: more methods are needed for this class.
+
+ // TODO: API needs to be cleaned up.
+
+ /**
+ * Constructs a row key for the application table as follows:
+ * {@code clusterId!userName!flowId!flowRunId!AppId}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @param appId
+ * @return byte array with the row key
+ */
+ public static byte[] getRowKey(String clusterId, String userId,
+ String flowId, Long flowRunId, String appId) {
+ byte[] first =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId, userId,
+ flowId));
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ byte[] second = Bytes.toBytes(ApplicationRowKey.invert(flowRunId));
+ byte[] third = Bytes.toBytes(appId);
+ return Separator.QUALIFIERS.join(first, second, third);
+ }
+
+ /**
+ * Converts a timestamp into its inverse timestamp to be used in (row) keys
+ * where we want to have the most recent timestamp in the top of the table
+ * (scans start at the most recent timestamp first).
+ *
+ * @param key value to be inverted so that the latest version will be first in
+ * a scan.
+ * @return inverted long
+ */
+ public static long invert(Long key) {
+ return Long.MAX_VALUE - key;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
new file mode 100644
index 0000000..d2a2cb9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
+
+/**
+ * The application table as column families info, config and metrics. Info
+ * stores information about a YARN application entity, config stores
+ * configuration data of a YARN application, metrics stores the metrics of a
+ * YARN application. This table is entirely analogous to the entity table but
+ * created for better performance.
+ *
+ * Example application table record:
+ *
+ * <pre>
+ * |-------------------------------------------------------------------------|
+ * | Row | Column Family | Column Family| Column Family|
+ * | key | info | metrics | config |
+ * |-------------------------------------------------------------------------|
+ * | clusterId! | id:appId | metricId1: | configKey1: |
+ * | userName! | | metricValue1 | configValue1 |
+ * | flowId! | created_time: | @timestamp1 | |
+ * | flowRunId! | 1392993084018 | | configKey2: |
+ * | AppId | | metriciD1: | configValue2 |
+ * | | modified_time: | metricValue2 | |
+ * | | 1392995081012 | @timestamp2 | |
+ * | | | | |
+ * | | i!infoKey: | metricId2: | |
+ * | | infoValue | metricValue1 | |
+ * | | | @timestamp2 | |
+ * | | r!relatesToKey: | | |
+ * | | id3?id4?id5 | | |
+ * | | | | |
+ * | | s!isRelatedToKey: | | |
+ * | | id7?id9?id6 | | |
+ * | | | | |
+ * | | e!eventId?timestamp?infoKey: | | |
+ * | | eventInfoValue | | |
+ * | | | | |
+ * | | flowVersion: | | |
+ * | | versionValue | | |
+ * |-------------------------------------------------------------------------|
+ * </pre>
+ */
+public class ApplicationTable extends BaseTable<ApplicationTable> {
+ /** application prefix */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".application";
+
+ /** config param name that specifies the application table name */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /**
+ * config param name that specifies the TTL for metrics column family in
+ * application table
+ */
+ private static final String METRICS_TTL_CONF_NAME = PREFIX
+ + ".table.metrics.ttl";
+
+ /** default value for application table name */
+ private static final String DEFAULT_TABLE_NAME =
+ "timelineservice.application";
+
+ /** default TTL is 30 days for metrics timeseries */
+ private static final int DEFAULT_METRICS_TTL = 2592000;
+
+ /** default max number of versions */
+ private static final int DEFAULT_METRICS_MAX_VERSIONS = 1000;
+
+ private static final Log LOG = LogFactory.getLog(ApplicationTable.class);
+
+ public ApplicationTable() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
+ * (org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor applicationTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(ApplicationColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ applicationTableDescp.addFamily(infoCF);
+
+ HColumnDescriptor configCF =
+ new HColumnDescriptor(ApplicationColumnFamily.CONFIGS.getBytes());
+ configCF.setBloomFilterType(BloomType.ROWCOL);
+ configCF.setBlockCacheEnabled(true);
+ applicationTableDescp.addFamily(configCF);
+
+ HColumnDescriptor metricsCF =
+ new HColumnDescriptor(ApplicationColumnFamily.METRICS.getBytes());
+ applicationTableDescp.addFamily(metricsCF);
+ metricsCF.setBlockCacheEnabled(true);
+ // always keep 1 version (the latest)
+ metricsCF.setMinVersions(1);
+ metricsCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
+ metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
+ DEFAULT_METRICS_TTL));
+ applicationTableDescp
+ .setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ applicationTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(applicationTableDescp,
+ TimelineHBaseSchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+
+ /**
+ * @param metricsTTL time to live parameter for the metrics in this table.
+ * @param hbaseConf configuration in which to set the metrics TTL config
+ * variable.
+ */
+ public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
+ hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
new file mode 100644
index 0000000..c60e6f5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index 58272ab..c8485c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -157,7 +157,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
* org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
* #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
*/
- public <T> NavigableMap<String, NavigableMap<Long, T>>
+ public <V> NavigableMap<String, NavigableMap<Long, V>>
readResultsWithTimestamps(Result result) throws IOException {
return column.readResultsWithTimestamps(result, columnPrefixBytes);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index f657a14..9a8bd8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -40,36 +40,35 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* Example entity table record:
*
* <pre>
- * |--------------------------------------------------------------------|
- * | Row | Column Family | Column Family| Column Family|
- * | key | info | metrics | config |
- * |--------------------------------------------------------------------|
- * | userName! | id:entityId | metricId1: | configKey1: |
- * | clusterId! | | metricValue1 | configValue1 |
- * | flowId! | type:entityType | @timestamp1 | |
- * | flowRunId! | | | configKey2: |
- * | AppId! | created_time: | metriciD1: | configValue2 |
- * | entityType!| 1392993084018 | metricValue2 | |
- * | entityId | | @timestamp2 | |
- * | | modified_time: | | |
- * | | 1392995081012 | metricId2: | |
- * | | | metricValue1 | |
- * | | i!infoKey: | @timestamp2 | |
- * | | infoValue | | |
- * | | | | |
- * | | r!relatesToKey: | | |
- * | | id3?id4?id5 | | |
- * | | | | |
- * | | s!isRelatedToKey | | |
- * | | id7?id9?id6 | | |
- * | | | | |
- * | | e!eventId?eventInfoKey: | | |
- * | | eventInfoValue | | |
- * | | @timestamp | | |
- * | | | | |
- * | | flowVersion: | | |
- * | | versionValue | | |
- * |--------------------------------------------------------------------|
+ * |-------------------------------------------------------------------------|
+ * | Row | Column Family | Column Family| Column Family|
+ * | key | info | metrics | config |
+ * |-------------------------------------------------------------------------|
+ * | userName! | id:entityId | metricId1: | configKey1: |
+ * | clusterId! | | metricValue1 | configValue1 |
+ * | flowId! | type:entityType | @timestamp1 | |
+ * | flowRunId! | | | configKey2: |
+ * | AppId! | created_time: | metriciD1: | configValue2 |
+ * | entityType!| 1392993084018 | metricValue2 | |
+ * | entityId | | @timestamp2 | |
+ * | | modified_time: | | |
+ * | | 1392995081012 | metricId2: | |
+ * | | | metricValue1 | |
+ * | | i!infoKey: | @timestamp2 | |
+ * | | infoValue | | |
+ * | | | | |
+ * | | r!relatesToKey: | | |
+ * | | id3?id4?id5 | | |
+ * | | | | |
+ * | | s!isRelatedToKey | | |
+ * | | id7?id9?id6 | | |
+ * | | | | |
+ * | | e!eventId?timestamp?infoKey: | | |
+ * | | eventInfoValue | | |
+ * | | | | |
+ * | | flowVersion: | | |
+ * | | versionValue | | |
+ * |-------------------------------------------------------------------------|
* </pre>
*/
public class EntityTable extends BaseTable<EntityTable> {
[02/50] [abbrv] hadoop git commit: MAPREDUCE-6327. Made MR AM use
timeline service v2 API to write history events and counters. Contributed by
Junping Du.
Posted by sj...@apache.org.
MAPREDUCE-6327. Made MR AM use timeline service v2 API to write history events and counters. Contributed by Junping Du.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ffb56f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ffb56f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ffb56f6
Branch: refs/heads/YARN-2928-rebase
Commit: 0ffb56f63ed57ddd743e4e3fa018b9075fc7d507
Parents: 7f6642a
Author: Zhijie Shen <zj...@apache.org>
Authored: Tue Apr 21 16:31:33 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:39:24 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 15 ++
.../jobhistory/JobHistoryEventHandler.java | 258 ++++++++++++++++---
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 23 ++
.../v2/app/rm/RMContainerAllocator.java | 9 +
.../hadoop/mapreduce/jobhistory/TestEvents.java | 8 +-
.../jobhistory/TestJobHistoryEventHandler.java | 9 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 5 +
.../mapreduce/jobhistory/AMStartedEvent.java | 18 ++
.../mapreduce/jobhistory/HistoryEvent.java | 4 +
.../mapreduce/jobhistory/JobFinishedEvent.java | 25 ++
.../jobhistory/JobInfoChangeEvent.java | 11 +
.../mapreduce/jobhistory/JobInitedEvent.java | 14 +
.../jobhistory/JobPriorityChangeEvent.java | 10 +
.../jobhistory/JobQueueChangeEvent.java | 10 +
.../jobhistory/JobStatusChangedEvent.java | 10 +
.../mapreduce/jobhistory/JobSubmittedEvent.java | 23 ++
.../JobUnsuccessfulCompletionEvent.java | 16 ++
.../jobhistory/MapAttemptFinishedEvent.java | 24 +-
.../jobhistory/NormalizedResourceEvent.java | 11 +
.../jobhistory/ReduceAttemptFinishedEvent.java | 25 +-
.../jobhistory/TaskAttemptFinishedEvent.java | 19 ++
.../jobhistory/TaskAttemptStartedEvent.java | 18 ++
.../TaskAttemptUnsuccessfulCompletionEvent.java | 24 ++
.../mapreduce/jobhistory/TaskFailedEvent.java | 19 ++
.../mapreduce/jobhistory/TaskFinishedEvent.java | 18 ++
.../mapreduce/jobhistory/TaskStartedEvent.java | 12 +
.../mapreduce/jobhistory/TaskUpdatedEvent.java | 10 +
.../mapreduce/util/JobHistoryEventUtils.java | 51 ++++
.../src/main/resources/mapred-default.xml | 7 +
.../mapred/TestMRTimelineEventHandling.java | 163 +++++++++++-
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 21 +-
31 files changed, 838 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index c98ea50..41bde85 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -1,5 +1,20 @@
Hadoop MapReduce Change Log
+Branch YARN-2928: Timeline Server Next Generation: Phase 1
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ MAPREDUCE-6327. Made MR AM use timeline service v2 API to write history
+ events and counters. (Junping Du via zjshen)
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Trunk (Unreleased)
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index dd0de2a..f52e654 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.mapreduce.jobhistory;
import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@@ -49,11 +52,13 @@ import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
+import org.apache.hadoop.mapreduce.v2.app.MRAppMaster;
import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
@@ -68,10 +73,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.node.ArrayNode;
-import org.codehaus.jackson.node.ObjectNode;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.common.annotations.VisibleForTesting;
/**
@@ -119,14 +122,24 @@ public class JobHistoryEventHandler extends AbstractService
protected static final Map<JobId, MetaInfo> fileMap =
Collections.<JobId,MetaInfo>synchronizedMap(new HashMap<JobId,MetaInfo>());
+
+ // For posting entities in new timeline service in a non-blocking way
+ // TODO YARN-3367 replace with event loop in TimelineClient.
+ private static ExecutorService threadPool =
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
+ .build());
// should job completion be force when the AM shuts down?
protected volatile boolean forceJobCompletion = false;
protected TimelineClient timelineClient;
+
+ private boolean newTimelineServiceEnabled = false;
private static String MAPREDUCE_JOB_ENTITY_TYPE = "MAPREDUCE_JOB";
private static String MAPREDUCE_TASK_ENTITY_TYPE = "MAPREDUCE_TASK";
+ private static String MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE = "MAPREDUCE_TASK_ATTEMPT";
public JobHistoryEventHandler(AppContext context, int startCount) {
super("JobHistoryEventHandler");
@@ -246,13 +259,22 @@ public class JobHistoryEventHandler extends AbstractService
MRJobConfig.MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD,
MRJobConfig.DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD);
+ // TODO replace MR specific configurations on timeline service with getting
+ // configuration from RM through registerApplicationMaster() in
+ // ApplicationMasterProtocol with return value for timeline service
+ // configuration status: off, on_with_v1 or on_with_v2.
if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA,
MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)) {
if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
- timelineClient = TimelineClient.createTimelineClient();
+
+ timelineClient =
+ ((MRAppMaster.RunningAppContext)context).getTimelineClient();
timelineClient.init(conf);
- LOG.info("Timeline service is enabled");
+ newTimelineServiceEnabled = conf.getBoolean(
+ MRJobConfig.MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED,
+ MRJobConfig.DEFAULT_MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED);
+ LOG.info("Timeline service is enabled: " + (newTimelineServiceEnabled? "v2" : "v1"));
LOG.info("Emitting job history data to the timeline server is enabled");
} else {
LOG.info("Timeline service is not enabled");
@@ -426,9 +448,26 @@ public class JobHistoryEventHandler extends AbstractService
if (timelineClient != null) {
timelineClient.stop();
}
+ shutdownAndAwaitTermination();
LOG.info("Stopped JobHistoryEventHandler. super.stop()");
super.serviceStop();
}
+
+ // TODO remove threadPool after adding non-blocking call in TimelineClient
+ private static void shutdownAndAwaitTermination() {
+ threadPool.shutdown();
+ try {
+ if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
+ threadPool.shutdownNow();
+ if (!threadPool.awaitTermination(60, TimeUnit.SECONDS))
+ LOG.error("ThreadPool did not terminate");
+ }
+ } catch (InterruptedException ie) {
+ threadPool.shutdownNow();
+ // Preserve interrupt status
+ Thread.currentThread().interrupt();
+ }
+ }
protected EventWriter createEventWriter(Path historyFilePath)
throws IOException {
@@ -583,8 +622,13 @@ public class JobHistoryEventHandler extends AbstractService
processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(),
event.getJobID());
if (timelineClient != null) {
- processEventForTimelineServer(historyEvent, event.getJobID(),
- event.getTimestamp());
+ if (newTimelineServiceEnabled) {
+ processEventForNewTimelineService(historyEvent, event.getJobID(),
+ event.getTimestamp());
+ } else {
+ processEventForTimelineServer(historyEvent, event.getJobID(),
+ event.getTimestamp());
+ }
}
if (LOG.isDebugEnabled()) {
LOG.debug("In HistoryEventHandler "
@@ -825,11 +869,11 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("FINISHED_MAPS", jfe.getFinishedMaps());
tEvent.addEventInfo("FINISHED_REDUCES", jfe.getFinishedReduces());
tEvent.addEventInfo("MAP_COUNTERS_GROUPS",
- countersToJSON(jfe.getMapCounters()));
+ JobHistoryEventUtils.countersToJSON(jfe.getMapCounters()));
tEvent.addEventInfo("REDUCE_COUNTERS_GROUPS",
- countersToJSON(jfe.getReduceCounters()));
+ JobHistoryEventUtils.countersToJSON(jfe.getReduceCounters()));
tEvent.addEventInfo("TOTAL_COUNTERS_GROUPS",
- countersToJSON(jfe.getTotalCounters()));
+ JobHistoryEventUtils.countersToJSON(jfe.getTotalCounters()));
tEvent.addEventInfo("JOB_STATUS", JobState.SUCCEEDED.toString());
tEntity.addEvent(tEvent);
tEntity.setEntityId(jobId.toString());
@@ -855,7 +899,7 @@ public class JobHistoryEventHandler extends AbstractService
tfe.getFailedAttemptID() == null ?
"" : tfe.getFailedAttemptID().toString());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(tfe.getCounters()));
+ JobHistoryEventUtils.countersToJSON(tfe.getCounters()));
tEntity.addEvent(tEvent);
tEntity.setEntityId(tfe.getTaskId().toString());
tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
@@ -873,7 +917,7 @@ public class JobHistoryEventHandler extends AbstractService
TaskFinishedEvent tfe2 = (TaskFinishedEvent) event;
tEvent.addEventInfo("TASK_TYPE", tfe2.getTaskType().toString());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(tfe2.getCounters()));
+ JobHistoryEventUtils.countersToJSON(tfe2.getCounters()));
tEvent.addEventInfo("FINISH_TIME", tfe2.getFinishTime());
tEvent.addEventInfo("STATUS", TaskStatus.State.SUCCEEDED.toString());
tEvent.addEventInfo("SUCCESSFUL_TASK_ATTEMPT_ID",
@@ -895,7 +939,6 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("START_TIME", tase.getStartTime());
tEvent.addEventInfo("HTTP_PORT", tase.getHttpPort());
tEvent.addEventInfo("TRACKER_NAME", tase.getTrackerName());
- tEvent.addEventInfo("TASK_TYPE", tase.getTaskType().toString());
tEvent.addEventInfo("SHUFFLE_PORT", tase.getShufflePort());
tEvent.addEventInfo("CONTAINER_ID", tase.getContainerId() == null ?
"" : tase.getContainerId().toString());
@@ -928,7 +971,7 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("SORT_FINISH_TIME", tauce.getFinishTime());
tEvent.addEventInfo("MAP_FINISH_TIME", tauce.getFinishTime());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(tauce.getCounters()));
+ JobHistoryEventUtils.countersToJSON(tauce.getCounters()));
tEntity.addEvent(tEvent);
tEntity.setEntityId(tauce.getTaskId().toString());
tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
@@ -942,7 +985,7 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("STATE", mafe.getState());
tEvent.addEventInfo("MAP_FINISH_TIME", mafe.getMapFinishTime());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(mafe.getCounters()));
+ JobHistoryEventUtils.countersToJSON(mafe.getCounters()));
tEvent.addEventInfo("HOSTNAME", mafe.getHostname());
tEvent.addEventInfo("PORT", mafe.getPort());
tEvent.addEventInfo("RACK_NAME", mafe.getRackName());
@@ -964,7 +1007,7 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("SHUFFLE_FINISH_TIME", rafe.getShuffleFinishTime());
tEvent.addEventInfo("SORT_FINISH_TIME", rafe.getSortFinishTime());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(rafe.getCounters()));
+ JobHistoryEventUtils.countersToJSON(rafe.getCounters()));
tEvent.addEventInfo("HOSTNAME", rafe.getHostname());
tEvent.addEventInfo("PORT", rafe.getPort());
tEvent.addEventInfo("RACK_NAME", rafe.getRackName());
@@ -983,7 +1026,7 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("STATUS", tafe.getTaskStatus());
tEvent.addEventInfo("STATE", tafe.getState());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(tafe.getCounters()));
+ JobHistoryEventUtils.countersToJSON(tafe.getCounters()));
tEvent.addEventInfo("HOSTNAME", tafe.getHostname());
tEntity.addEvent(tEvent);
tEntity.setEntityId(tafe.getTaskId().toString());
@@ -1010,37 +1053,172 @@ public class JobHistoryEventHandler extends AbstractService
default:
break;
}
-
+
try {
timelineClient.putEntities(tEntity);
- } catch (IOException ex) {
- LOG.error("Error putting entity " + tEntity.getEntityId() + " to Timeline"
- + "Server", ex);
- } catch (YarnException ex) {
+ } catch (IOException|YarnException ex) {
LOG.error("Error putting entity " + tEntity.getEntityId() + " to Timeline"
+ "Server", ex);
}
}
-
- @Private
- public JsonNode countersToJSON(Counters counters) {
- ObjectMapper mapper = new ObjectMapper();
- ArrayNode nodes = mapper.createArrayNode();
- if (counters != null) {
- for (CounterGroup counterGroup : counters) {
- ObjectNode groupNode = nodes.addObject();
- groupNode.put("NAME", counterGroup.getName());
- groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName());
- ArrayNode countersNode = groupNode.putArray("COUNTERS");
- for (Counter counter : counterGroup) {
- ObjectNode counterNode = countersNode.addObject();
- counterNode.put("NAME", counter.getName());
- counterNode.put("DISPLAY_NAME", counter.getDisplayName());
- counterNode.put("VALUE", counter.getValue());
+
+ private void putEntityWithoutBlocking(final TimelineClient timelineClient,
+ final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity) {
+ Runnable publishWrapper = new Runnable() {
+ public void run() {
+ try {
+ timelineClient.putEntities(entity);
+ } catch (IOException|YarnException e) {
+ LOG.error("putEntityNonBlocking get failed: " + e);
+ throw new RuntimeException(e.toString());
}
}
+ };
+ threadPool.execute(publishWrapper);
+ }
+
+ // create JobEntity from HistoryEvent with adding other info, like:
+ // jobId, timestamp and entityType.
+ private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ createJobEntity(HistoryEvent event, long timestamp, JobId jobId,
+ String entityType) {
+
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType);
+ entity.setId(jobId.toString());
+ return entity;
+ }
+
+ // create BaseEntity from HistoryEvent with adding other info, like:
+ // timestamp and entityType.
+ private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ createBaseEntity(HistoryEvent event, long timestamp, String entityType) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent =
+ event.toTimelineEvent();
+ tEvent.setTimestamp(timestamp);
+
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
+ entity.addEvent(tEvent);
+ entity.setType(entityType);
+ return entity;
+ }
+
+ // create TaskEntity from HistoryEvent with adding other info, like:
+ // taskId, jobId, timestamp, entityType and relatedJobEntity.
+ private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ createTaskEntity(HistoryEvent event, long timestamp, String taskId,
+ String entityType, String relatedJobEntity, JobId jobId) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType);
+ entity.setId(taskId);
+ entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString());
+ return entity;
+ }
+
+ // create TaskAttemptEntity from HistoryEvent with adding other info, like:
+ // timestamp, taskAttemptId, entityType, relatedTaskEntity and taskId.
+ private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ createTaskAttemptEntity(HistoryEvent event, long timestamp,
+ String taskAttemptId, String entityType, String relatedTaskEntity,
+ String taskId) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType);
+ entity.setId(taskAttemptId);
+ entity.addIsRelatedToEntity(relatedTaskEntity, taskId);
+ return entity;
+ }
+
+ private void processEventForNewTimelineService(HistoryEvent event, JobId jobId,
+ long timestamp) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity tEntity = null;
+ String taskId = null;
+ String taskAttemptId = null;
+
+ switch (event.getEventType()) {
+ // Handle job events
+ case JOB_SUBMITTED:
+ case JOB_STATUS_CHANGED:
+ case JOB_INFO_CHANGED:
+ case JOB_INITED:
+ case JOB_PRIORITY_CHANGED:
+ case JOB_QUEUE_CHANGED:
+ case JOB_FAILED:
+ case JOB_KILLED:
+ case JOB_ERROR:
+ case JOB_FINISHED:
+ case AM_STARTED:
+ case NORMALIZED_RESOURCE:
+ break;
+ // Handle task events
+ case TASK_STARTED:
+ taskId = ((TaskStartedEvent)event).getTaskId().toString();
+ break;
+ case TASK_FAILED:
+ taskId = ((TaskFailedEvent)event).getTaskId().toString();
+ break;
+ case TASK_UPDATED:
+ taskId = ((TaskUpdatedEvent)event).getTaskId().toString();
+ break;
+ case TASK_FINISHED:
+ taskId = ((TaskFinishedEvent)event).getTaskId().toString();
+ break;
+ case MAP_ATTEMPT_STARTED:
+ case CLEANUP_ATTEMPT_STARTED:
+ case REDUCE_ATTEMPT_STARTED:
+ case SETUP_ATTEMPT_STARTED:
+ taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptStartedEvent)event).
+ getTaskAttemptId().toString();
+ break;
+ case MAP_ATTEMPT_FAILED:
+ case CLEANUP_ATTEMPT_FAILED:
+ case REDUCE_ATTEMPT_FAILED:
+ case SETUP_ATTEMPT_FAILED:
+ case MAP_ATTEMPT_KILLED:
+ case CLEANUP_ATTEMPT_KILLED:
+ case REDUCE_ATTEMPT_KILLED:
+ case SETUP_ATTEMPT_KILLED:
+ taskId = ((TaskAttemptUnsuccessfulCompletionEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptUnsuccessfulCompletionEvent)event).
+ getTaskAttemptId().toString();
+ break;
+ case MAP_ATTEMPT_FINISHED:
+ taskId = ((MapAttemptFinishedEvent)event).getTaskId().toString();
+ taskAttemptId = ((MapAttemptFinishedEvent)event).getAttemptId().toString();
+ break;
+ case REDUCE_ATTEMPT_FINISHED:
+ taskId = ((ReduceAttemptFinishedEvent)event).getTaskId().toString();
+ taskAttemptId = ((ReduceAttemptFinishedEvent)event).getAttemptId().toString();
+ break;
+ case SETUP_ATTEMPT_FINISHED:
+ case CLEANUP_ATTEMPT_FINISHED:
+ taskId = ((TaskAttemptFinishedEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptFinishedEvent)event).getAttemptId().toString();
+ break;
+ default:
+ LOG.warn("EventType: " + event.getEventType() + " cannot be recognized" +
+ " and handled by timeline service.");
+ return;
}
- return nodes;
+ if (taskId == null) {
+ // JobEntity
+ tEntity = createJobEntity(event, timestamp, jobId,
+ MAPREDUCE_JOB_ENTITY_TYPE);
+ } else {
+ if (taskAttemptId == null) {
+ // TaskEntity
+ tEntity = createTaskEntity(event, timestamp, taskId,
+ MAPREDUCE_TASK_ENTITY_TYPE, MAPREDUCE_JOB_ENTITY_TYPE, jobId);
+ } else {
+ // TaskAttemptEntity
+ tEntity = createTaskAttemptEntity(event, timestamp, taskAttemptId,
+ MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE, MAPREDUCE_TASK_ENTITY_TYPE,
+ taskId);
+ }
+ }
+
+ putEntityWithoutBlocking(timelineClient, tEntity);
}
private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 6dc830f..dafb6e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -135,6 +135,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -1004,6 +1005,7 @@ public class MRAppMaster extends CompositeService {
private final Configuration conf;
private final ClusterInfo clusterInfo = new ClusterInfo();
private final ClientToAMTokenSecretManager clientToAMTokenSecretManager;
+ private TimelineClient timelineClient = null;
private final TaskAttemptFinishingMonitor taskAttemptFinishingMonitor;
@@ -1013,6 +1015,23 @@ public class MRAppMaster extends CompositeService {
this.clientToAMTokenSecretManager =
new ClientToAMTokenSecretManager(appAttemptID, null);
this.taskAttemptFinishingMonitor = taskAttemptFinishingMonitor;
+ if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA,
+ MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)
+ && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+
+ boolean newTimelineServiceEnabled = conf.getBoolean(
+ MRJobConfig.MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED,
+ MRJobConfig.DEFAULT_MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED);
+
+ if (newTimelineServiceEnabled) {
+ // create new version TimelineClient
+ timelineClient = TimelineClient.createTimelineClient(
+ appAttemptID.getApplicationId());
+ } else {
+ timelineClient = TimelineClient.createTimelineClient();
+ }
+ }
}
@Override
@@ -1103,6 +1122,10 @@ public class MRAppMaster extends CompositeService {
return taskAttemptFinishingMonitor;
}
+ // Get Timeline Collector's address (get sync from RM)
+ public TimelineClient getTimelineClient() {
+ return timelineClient;
+ }
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index bf9b1f8..c721341 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.MRAppMaster;
import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
@@ -797,6 +798,14 @@ public class RMContainerAllocator extends RMContainerRequestor
computeIgnoreBlacklisting();
handleUpdatedNodes(response);
+ String collectorAddr = response.getCollectorAddr();
+ MRAppMaster.RunningAppContext appContext =
+ (MRAppMaster.RunningAppContext)this.getContext();
+ if (collectorAddr != null && !collectorAddr.isEmpty()
+ && appContext.getTimelineClient() != null) {
+ appContext.getTimelineClient().setTimelineServiceAddress(
+ response.getCollectorAddr());
+ }
for (ContainerStatus cont : finishedContainers) {
LOG.info("Received completed container " + cont.getContainerId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
index 7612ceb..e7d5006 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.junit.Test;
public class TestEvents {
@@ -404,7 +405,12 @@ public class TestEvents {
public void setDatum(Object datum) {
this.datum = datum;
}
-
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ return null;
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 2b07efb..2c159f1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
@@ -666,7 +667,7 @@ public class TestJobHistoryEventHandler {
group2.addCounter("MARTHA_JONES", "Martha Jones", 3);
group2.addCounter("DONNA_NOBLE", "Donna Noble", 2);
group2.addCounter("ROSE_TYLER", "Rose Tyler", 1);
- JsonNode jsonNode = jheh.countersToJSON(counters);
+ JsonNode jsonNode = JobHistoryEventUtils.countersToJSON(counters);
String jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
String expected = "[{\"NAME\":\"COMPANIONS\",\"DISPLAY_NAME\":\"Companions "
+ "of the Doctor\",\"COUNTERS\":[{\"NAME\":\"AMY_POND\",\"DISPLAY_NAME\""
@@ -689,19 +690,19 @@ public class TestJobHistoryEventHandler {
public void testCountersToJSONEmpty() throws Exception {
JobHistoryEventHandler jheh = new JobHistoryEventHandler(null, 0);
Counters counters = null;
- JsonNode jsonNode = jheh.countersToJSON(counters);
+ JsonNode jsonNode = JobHistoryEventUtils.countersToJSON(counters);
String jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
String expected = "[]";
Assert.assertEquals(expected, jsonStr);
counters = new Counters();
- jsonNode = jheh.countersToJSON(counters);
+ jsonNode = JobHistoryEventUtils.countersToJSON(counters);
jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
expected = "[]";
Assert.assertEquals(expected, jsonStr);
counters.addGroup("DOCTORS", "Incarnations of the Doctor");
- jsonNode = jheh.countersToJSON(counters);
+ jsonNode = JobHistoryEventUtils.countersToJSON(counters);
jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
expected = "[{\"NAME\":\"DOCTORS\",\"DISPLAY_NAME\":\"Incarnations of the "
+ "Doctor\",\"COUNTERS\":[]}]";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index e321817..c06ea05 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -462,6 +462,11 @@ public interface MRJobConfig {
"mapreduce.job.emit-timeline-data";
public static final boolean DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA =
false;
+
+ public static final String MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED =
+ "mapreduce.job.new-timeline-service.enabled";
+ public static final boolean DEFAULT_MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED =
+ false;
public static final String MR_PREFIX = "yarn.app.mapreduce.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
index ea2ca9e..bbc7090 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
@@ -20,8 +20,10 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.avro.util.Utf8;
@@ -166,4 +168,20 @@ public class AMStartedEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.AM_STARTED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("APPLICATION_ATTEMPT_ID",
+ getAppAttemptId() == null ? "" : getAppAttemptId().toString());
+ tEvent.addInfo("CONTAINER_ID", getContainerId() == null ?
+ "" : getContainerId().toString());
+ tEvent.addInfo("NODE_MANAGER_HOST", getNodeManagerHost());
+ tEvent.addInfo("NODE_MANAGER_PORT", getNodeManagerPort());
+ tEvent.addInfo("NODE_MANAGER_HTTP_PORT", getNodeManagerHttpPort());
+ tEvent.addInfo("START_TIME", getStartTime());
+ return tEvent;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
index a30748c..61ce217 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Interface for event wrapper classes. Implementations each wrap an
@@ -37,4 +38,7 @@ public interface HistoryEvent {
/** Set the Avro datum wrapped by this. */
void setDatum(Object datum);
+
+ /** Map HistoryEvent to TimelineEvent */
+ TimelineEvent toTimelineEvent();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
index 0fa5868..80d3ee6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
@@ -23,6 +23,9 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record successful completion of job
@@ -133,4 +136,26 @@ public class JobFinishedEvent implements HistoryEvent {
public Counters getReduceCounters() {
return reduceCounters;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("NUM_MAPS", getFinishedMaps());
+ tEvent.addInfo("NUM_REDUCES", getFinishedReduces());
+ tEvent.addInfo("FAILED_MAPS", getFailedMaps());
+ tEvent.addInfo("FAILED_REDUCES", getFailedReduces());
+ tEvent.addInfo("FINISHED_MAPS", getFinishedMaps());
+ tEvent.addInfo("FINISHED_REDUCES", getFinishedReduces());
+ tEvent.addInfo("MAP_COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getMapCounters()));
+ tEvent.addInfo("REDUCE_COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getReduceCounters()));
+ tEvent.addInfo("TOTAL_COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getTotalCounters()));
+ // TODO replace SUCCEEDED with JobState.SUCCEEDED.toString()
+ tEvent.addInfo("JOB_STATUS", "SUCCEEDED");
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
index b45f373..ad82443 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -64,5 +66,14 @@ public class JobInfoChangeEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.JOB_INFO_CHANGED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("SUBMIT_TIME", getSubmitTime());
+ tEvent.addInfo("LAUNCH_TIME", getLaunchTime());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
index 5abb40e..3e0f2f7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -73,4 +75,16 @@ public class JobInitedEvent implements HistoryEvent {
}
/** Get whether the job's map and reduce stages were combined */
public boolean getUberized() { return datum.getUberized(); }
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("START_TIME", getLaunchTime());
+ tEvent.addInfo("STATUS", getStatus());
+ tEvent.addInfo("TOTAL_MAPS", getTotalMaps());
+ tEvent.addInfo("TOTAL_REDUCES", getTotalReduces());
+ tEvent.addInfo("UBERIZED", getUberized());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
index 6c32462..5deea0a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.JobPriority;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -64,5 +66,13 @@ public class JobPriorityChangeEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.JOB_PRIORITY_CHANGED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("PRIORITY", getPriority().toString());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
index 86078e6..b9dd359 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
@SuppressWarnings("deprecation")
public class JobQueueChangeEvent implements HistoryEvent {
@@ -59,5 +61,13 @@ public class JobQueueChangeEvent implements HistoryEvent {
}
return null;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("QUEUE_NAMES", getJobQueueName());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
index 9e11a55..a4f2da2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -60,5 +62,13 @@ public class JobStatusChangedEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.JOB_STATUS_CHANGED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("STATUS", getStatus());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
index 845f6f7..47b2840 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -205,5 +207,26 @@ public class JobSubmittedEvent implements HistoryEvent {
}
/** Get the event type */
public EventType getEventType() { return EventType.JOB_SUBMITTED; }
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("SUBMIT_TIME", getSubmitTime());
+ tEvent.addInfo("QUEUE_NAME", getJobQueueName());
+ tEvent.addInfo("JOB_NAME", getJobName());
+ tEvent.addInfo("USER_NAME", getUserName());
+ tEvent.addInfo("JOB_CONF_PATH", getJobConfPath());
+ tEvent.addInfo("ACLS", getJobAcls());
+ tEvent.addInfo("JOB_QUEUE_NAME", getJobQueueName());
+ tEvent.addInfo("WORKLFOW_ID", getWorkflowId());
+ tEvent.addInfo("WORKFLOW_NAME", getWorkflowName());
+ tEvent.addInfo("WORKFLOW_NODE_NAME", getWorkflowNodeName());
+ tEvent.addInfo("WORKFLOW_ADJACENCIES",
+ getWorkflowAdjacencies());
+ tEvent.addInfo("WORKFLOW_TAGS", getWorkflowTags());
+
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
index 1b7773d..ea9798c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
@@ -24,6 +24,8 @@ import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import java.util.Collections;
@@ -119,4 +121,18 @@ public class JobUnsuccessfulCompletionEvent implements HistoryEvent {
final CharSequence diagnostics = datum.getDiagnostics();
return diagnostics == null ? NODIAGS : diagnostics.toString();
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("NUM_MAPS", getFinishedMaps());
+ tEvent.addInfo("NUM_REDUCES", getFinishedReduces());
+ tEvent.addInfo("JOB_STATUS", getStatus());
+ tEvent.addInfo("DIAGNOSTICS", getDiagnostics());
+ tEvent.addInfo("FINISHED_MAPS", getFinishedMaps());
+ tEvent.addInfo("FINISHED_REDUCES", getFinishedReduces());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
index bc046c7..36737e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
@@ -26,6 +26,9 @@ import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record successful completion of a map attempt
@@ -33,7 +36,7 @@ import org.apache.hadoop.mapreduce.TaskType;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class MapAttemptFinishedEvent implements HistoryEvent {
+public class MapAttemptFinishedEvent implements HistoryEvent {
private MapAttemptFinished datum = null;
@@ -218,4 +221,23 @@ public class MapAttemptFinishedEvent implements HistoryEvent {
return physMemKbytes;
}
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("STATUS", getTaskStatus());
+ tEvent.addInfo("STATE", getState());
+ tEvent.addInfo("MAP_FINISH_TIME", getMapFinishTime());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ tEvent.addInfo("HOSTNAME", getHostname());
+ tEvent.addInfo("PORT", getPort());
+ tEvent.addInfo("RACK_NAME", getRackName());
+ tEvent.addInfo("ATTEMPT_ID", getAttemptId() == null ?
+ "" : getAttemptId().toString());
+ return tEvent;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
index b8f049c..daa454c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record the normalized map/reduce requirements.
@@ -71,4 +73,13 @@ public class NormalizedResourceEvent implements HistoryEvent {
public void setDatum(Object datum) {
throw new UnsupportedOperationException("Not a seriable object");
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("MEMORY", "" + getMemory());
+ tEvent.addInfo("TASK_TYPE", getTaskType());
+ return tEvent;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
index 6644a48..6087c7a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
@@ -26,6 +26,9 @@ import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record successful completion of a reduce attempt
@@ -33,7 +36,7 @@ import org.apache.hadoop.mapreduce.TaskType;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class ReduceAttemptFinishedEvent implements HistoryEvent {
+public class ReduceAttemptFinishedEvent implements HistoryEvent {
private ReduceAttemptFinished datum = null;
@@ -222,5 +225,25 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent {
public int[] getPhysMemKbytes() {
return physMemKbytes;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("ATTEMPT_ID", getAttemptId() == null ?
+ "" : getAttemptId().toString());
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("STATUS", getTaskStatus());
+ tEvent.addInfo("STATE", getState());
+ tEvent.addInfo("SHUFFLE_FINISH_TIME", getShuffleFinishTime());
+ tEvent.addInfo("SORT_FINISH_TIME", getSortFinishTime());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ tEvent.addInfo("HOSTNAME", getHostname());
+ tEvent.addInfo("PORT", getPort());
+ tEvent.addInfo("RACK_NAME", getRackName());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
index bb7dbe0..c7c4387 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
@@ -25,6 +25,9 @@ import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record successful task completion
@@ -135,5 +138,21 @@ public class TaskAttemptFinishedEvent implements HistoryEvent {
? EventType.MAP_ATTEMPT_FINISHED
: EventType.REDUCE_ATTEMPT_FINISHED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("ATTEMPT_ID", getAttemptId() == null ?
+ "" : getAttemptId().toString());
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("STATUS", getTaskStatus());
+ tEvent.addInfo("STATE", getState());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ tEvent.addInfo("HOSTNAME", getHostname());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
index c8c250a..d8baec4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
@@ -23,8 +23,10 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -132,5 +134,21 @@ public class TaskAttemptStartedEvent implements HistoryEvent {
}
return null;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("TASK_ATTEMPT_ID",
+ getTaskAttemptId().toString());
+ tEvent.addInfo("START_TIME", getStartTime());
+ tEvent.addInfo("HTTP_PORT", getHttpPort());
+ tEvent.addInfo("TRACKER_NAME", getTrackerName());
+ tEvent.addInfo("SHUFFLE_PORT", getShufflePort());
+ tEvent.addInfo("CONTAINER_ID", getContainerId() == null ?
+ "" : getContainerId().toString());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
index 77ee2a0..0bb1358 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
@@ -25,6 +25,9 @@ import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.mapred.ProgressSplitsBlock;
@@ -247,5 +250,26 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent {
public int[] getPhysMemKbytes() {
return physMemKbytes;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("TASK_ATTEMPT_ID", getTaskAttemptId() == null ?
+ "" : getTaskAttemptId().toString());
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("ERROR", getError());
+ tEvent.addInfo("STATUS", getTaskStatus());
+ tEvent.addInfo("HOSTNAME", getHostname());
+ tEvent.addInfo("PORT", getPort());
+ tEvent.addInfo("RACK_NAME", getRackName());
+ tEvent.addInfo("SHUFFLE_FINISH_TIME", getFinishTime());
+ tEvent.addInfo("SORT_FINISH_TIME", getFinishTime());
+ tEvent.addInfo("MAP_FINISH_TIME", getFinishTime());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
index 2838f08..5e82dea 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
@@ -20,10 +20,14 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapred.TaskStatus;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -136,5 +140,20 @@ public class TaskFailedEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.TASK_FAILED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("STATUS", TaskStatus.State.FAILED.toString());
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("ERROR", getError());
+ tEvent.addInfo("FAILED_ATTEMPT_ID",
+ getFailedAttemptID() == null ? "" : getFailedAttemptID().toString());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
index d4ec74d..e359e32 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
@@ -21,10 +21,14 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapred.TaskStatus;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record the successful completion of a task
@@ -115,5 +119,19 @@ public class TaskFinishedEvent implements HistoryEvent {
return EventType.TASK_FINISHED;
}
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("STATUS", TaskStatus.State.SUCCEEDED.toString());
+ tEvent.addInfo("SUCCESSFUL_TASK_ATTEMPT_ID",
+ getSuccessfulTaskAttemptId() == null ? "" :
+ getSuccessfulTaskAttemptId().toString());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
index ed53b03..d1b97bf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record the start of a task
@@ -71,5 +73,15 @@ public class TaskStartedEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.TASK_STARTED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("START_TIME", getStartTime());
+ tEvent.addInfo("SPLIT_LOCATIONS", getSplitLocations());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
index 58f4143..b9a389c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -60,5 +62,13 @@ public class TaskUpdatedEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.TASK_UPDATED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
new file mode 100644
index 0000000..f4896ff
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.mapreduce.util;
+
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
+
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.node.ArrayNode;
+import org.codehaus.jackson.node.ObjectNode;
+
+public class JobHistoryEventUtils {
+
+ public static JsonNode countersToJSON(Counters counters) {
+ ObjectMapper mapper = new ObjectMapper();
+ ArrayNode nodes = mapper.createArrayNode();
+ if (counters != null) {
+ for (CounterGroup counterGroup : counters) {
+ ObjectNode groupNode = nodes.addObject();
+ groupNode.put("NAME", counterGroup.getName());
+ groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName());
+ ArrayNode countersNode = groupNode.putArray("COUNTERS");
+ for (Counter counter : counterGroup) {
+ ObjectNode counterNode = countersNode.addObject();
+ counterNode.put("NAME", counter.getName());
+ counterNode.put("DISPLAY_NAME", counter.getDisplayName());
+ counterNode.put("VALUE", counter.getValue());
+ }
+ }
+ }
+ return nodes;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ffb56f6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 909f5c5..fba1c87 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -618,6 +618,13 @@
</description>
</property>
+ <property>
+ <name>mapreduce.job.new-timeline-service.enabled</name>
+ <value>false</value>
+ <description>Specifies if posting job and task events to new timeline service.
+ </description>
+</property>
+
<property>
<name>mapreduce.input.fileinputformat.split.minsize</name>
<value>0</value>
[06/50] [abbrv] hadoop git commit: YARN-3134. Implemented Phoenix
timeline writer to access HBase backend. Contributed by Li Lu.
Posted by sj...@apache.org.
YARN-3134. Implemented Phoenix timeline writer to access HBase backend. Contributed by Li Lu.
(cherry picked from commit b3b791be466be79e4e964ad068f7a6ec701e22e1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d103fbe9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d103fbe9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d103fbe9
Branch: refs/heads/YARN-2928-rebase
Commit: d103fbe934cf5bb940061ec7e5dc55c05398b143
Parents: 87823cf
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri May 8 19:08:02 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:48:37 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 11 +
.../hadoop-yarn-server-timelineservice/pom.xml | 17 +
.../collector/TimelineCollector.java | 13 +-
.../collector/TimelineCollectorManager.java | 19 +
.../storage/PhoenixTimelineWriterImpl.java | 509 +++++++++++++++++++
.../storage/TestPhoenixTimelineWriterImpl.java | 125 +++++
.../storage/TestTimelineWriterImpl.java | 74 +++
8 files changed, 760 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d103fbe9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 02d2065..a05e4f6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -64,6 +64,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3562. unit tests failures and issues found from findbug from earlier
ATS checkins (Naganarasimha G R via sjlee)
+ YARN-3134. Implemented Phoenix timeline writer to access HBase backend. (Li
+ Lu via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d103fbe9/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 114851f..d25d1d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -485,6 +485,17 @@
</Or>
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <!-- Ignore SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING warnings for Timeline Phoenix storage. -->
+ <!-- Since we're using dynamic columns, we have to generate SQL statements dynamically -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.timelineservice.storage.PhoenixTimelineWriterImpl" />
+ <Or>
+ <Method name="storeEntityVariableLengthFields" />
+ <Method name="storeEvents" />
+ <Method name="storeMetrics" />
+ <Method name="write" />
+ </Or>
+ </Match>
<!-- Following fields are used in ErrorsAndWarningsBlock, which is not a part of analysis of findbugs -->
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d103fbe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index f974aee..f62230f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -120,6 +120,23 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <version>4.3.0</version>
+ <exclusions>
+ <!-- Exclude jline from here -->
+ <exclusion>
+ <artifactId>jline</artifactId>
+ <groupId>jline</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d103fbe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
index 4eced5b..bb7db12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -27,11 +27,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
/**
@@ -55,11 +52,6 @@ public abstract class TimelineCollector extends CompositeService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
- writer = ReflectionUtils.newInstance(conf.getClass(
- YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
- FileSystemTimelineWriterImpl.class,
- TimelineWriter.class), conf);
- writer.init(conf);
}
@Override
@@ -70,11 +62,10 @@ public abstract class TimelineCollector extends CompositeService {
@Override
protected void serviceStop() throws Exception {
super.serviceStop();
- writer.stop();
}
- public TimelineWriter getWriter() {
- return writer;
+ protected void setWriter(TimelineWriter w) {
+ this.writer = w;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d103fbe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 7b3da6b..953d9b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -23,9 +23,14 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
import java.util.Collections;
import java.util.HashMap;
@@ -42,6 +47,19 @@ public abstract class TimelineCollectorManager extends AbstractService {
private static final Log LOG =
LogFactory.getLog(TimelineCollectorManager.class);
+ protected TimelineWriter writer;
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ writer = ReflectionUtils.newInstance(conf.getClass(
+ YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
+ FileSystemTimelineWriterImpl.class,
+ TimelineWriter.class), conf);
+ writer.init(conf);
+ super.serviceInit(conf);
+ }
+
+
// access to this map is synchronized with the map itself
private final Map<ApplicationId, TimelineCollector> collectors =
Collections.synchronizedMap(
@@ -69,6 +87,7 @@ public abstract class TimelineCollectorManager extends AbstractService {
// initialize, start, and add it to the collection so it can be
// cleaned up when the parent shuts down
collector.init(getConfig());
+ collector.setWriter(writer);
collector.start();
collectors.put(appId, collector);
LOG.info("the collector for " + appId + " was added");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d103fbe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
new file mode 100644
index 0000000..af8a233
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
@@ -0,0 +1,509 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+@Private
+@Unstable
+public class PhoenixTimelineWriterImpl extends AbstractService
+ implements TimelineWriter {
+
+ private static final Log LOG
+ = LogFactory.getLog(PhoenixTimelineWriterImpl.class);
+ private static final String PHOENIX_COL_FAMILY_PLACE_HOLDER
+ = "timeline_cf_placeholder";
+ // These lists are not taking effects in table creations.
+ private static final String[] PHOENIX_STORAGE_PK_LIST
+ = {"cluster", "user", "flow_name", "flow_version", "flow_run", "app_id",
+ "type", "entity_id"};
+ private static final String[] TIMELINE_EVENT_EXTRA_PK_LIST =
+ {"timestamp", "event_id"};
+ private static final String[] TIMELINE_METRIC_EXTRA_PK_LIST =
+ {"metric_id"};
+ /** Default Phoenix JDBC driver name */
+ private static final String DRIVER_CLASS_NAME
+ = "org.apache.phoenix.jdbc.PhoenixDriver";
+
+ /** Default Phoenix timeline entity table name */
+ @VisibleForTesting
+ static final String ENTITY_TABLE_NAME = "timeline_entity";
+ /** Default Phoenix event table name */
+ @VisibleForTesting
+ static final String EVENT_TABLE_NAME = "timeline_event";
+ /** Default Phoenix metric table name */
+ @VisibleForTesting
+ static final String METRIC_TABLE_NAME = "metric_singledata";
+
+ /** Default Phoenix timeline config column family */
+ private static final String CONFIG_COLUMN_FAMILY = "c.";
+ /** Default Phoenix timeline info column family */
+ private static final String INFO_COLUMN_FAMILY = "i.";
+ /** Default Phoenix event info column family */
+ private static final String EVENT_INFO_COLUMN_FAMILY = "ei.";
+ /** Default Phoenix isRelatedTo column family */
+ private static final String IS_RELATED_TO_FAMILY = "ir.";
+ /** Default Phoenix relatesTo column family */
+ private static final String RELATES_TO_FAMILY = "rt.";
+ /** Default separator for Phoenix storage */
+ private static final String PHOENIX_STORAGE_SEPARATOR = ";";
+
+ /** Connection string to the deployed Phoenix cluster */
+ static final String CONN_STRING = "jdbc:phoenix:localhost:2181:/hbase";
+
+ PhoenixTimelineWriterImpl() {
+ super((PhoenixTimelineWriterImpl.class.getName()));
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ createTables();
+ super.init(conf);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ }
+
+ @Override
+ public TimelineWriteResponse write(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRunId, String appId,
+ TimelineEntities entities) throws IOException {
+ TimelineWriteResponse response = new TimelineWriteResponse();
+ TimelineCollectorContext currContext = new TimelineCollectorContext(
+ clusterId, userId, flowName, flowVersion, flowRunId, appId);
+ String sql = "UPSERT INTO " + ENTITY_TABLE_NAME
+ + " (" + StringUtils.join(PHOENIX_STORAGE_PK_LIST, ",")
+ + ", creation_time, modified_time, configs) "
+ + "VALUES (" + StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length)
+ + "?, ?, ?)";
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("TimelineEntity write SQL: " + sql);
+ }
+
+ try (Connection conn = getConnection();
+ PreparedStatement ps = conn.prepareStatement(sql)) {
+ for (TimelineEntity entity : entities.getEntities()) {
+ int idx = setStringsForPrimaryKey(ps, currContext, entity, 1);
+ ps.setLong(idx++, entity.getCreatedTime());
+ ps.setLong(idx++, entity.getModifiedTime());
+ String configKeys = StringUtils.join(
+ entity.getConfigs().keySet(), PHOENIX_STORAGE_SEPARATOR);
+ ps.setString(idx++, configKeys);
+ ps.execute();
+
+ storeEntityVariableLengthFields(entity, currContext, conn);
+ storeEvents(entity, currContext, conn);
+ storeMetrics(entity, currContext, conn);
+
+ conn.commit();
+ }
+ } catch (SQLException se) {
+ LOG.error("Failed to add entity to Phoenix " + se.getMessage());
+ throw new IOException(se);
+ } catch (Exception e) {
+ LOG.error("Exception on getting connection: " + e.getMessage());
+ throw new IOException(e);
+ }
+ return response;
+ }
+
+ /**
+ * Aggregates the entity information to the timeline store based on which
+ * track this entity is to be rolled up to The tracks along which aggregations
+ * are to be done are given by {@link TimelineAggregationTrack}
+ *
+ * Any errors occurring for individual write request objects will be reported
+ * in the response.
+ *
+ * @param data
+ * a {@link TimelineEntity} object
+ * a {@link TimelineAggregationTrack} enum value
+ * @return a {@link TimelineWriteResponse} object.
+ * @throws IOException
+ */
+ @Override
+ public TimelineWriteResponse aggregate(TimelineEntity data,
+ TimelineAggregationTrack track) throws IOException {
+ return null;
+
+ }
+
+ // Utility functions
+ @Private
+ @VisibleForTesting
+ static Connection getConnection() throws IOException {
+ Connection conn;
+ try {
+ Class.forName(DRIVER_CLASS_NAME);
+ conn = DriverManager.getConnection(CONN_STRING);
+ conn.setAutoCommit(false);
+ } catch (SQLException se) {
+ LOG.error("Failed to connect to phoenix server! "
+ + se.getLocalizedMessage());
+ throw new IOException(se);
+ } catch (ClassNotFoundException e) {
+ LOG.error("Class not found! " + e.getLocalizedMessage());
+ throw new IOException(e);
+ }
+ return conn;
+ }
+
+ private void createTables() throws Exception {
+ // Create tables if necessary
+ try (Connection conn = getConnection();
+ Statement stmt = conn.createStatement()) {
+ // Table schema defined as in YARN-3134.
+ String sql = "CREATE TABLE IF NOT EXISTS " + ENTITY_TABLE_NAME
+ + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
+ + "flow_run UNSIGNED_LONG NOT NULL, "
+ + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
+ + "entity_id VARCHAR NOT NULL, "
+ + "creation_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
+ + "configs VARCHAR, "
+ + CONFIG_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR, "
+ + INFO_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ + IS_RELATED_TO_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR, "
+ + RELATES_TO_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR "
+ + "CONSTRAINT pk PRIMARY KEY("
+ + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
+ + "type, entity_id))";
+ stmt.executeUpdate(sql);
+ sql = "CREATE TABLE IF NOT EXISTS " + EVENT_TABLE_NAME
+ + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
+ + "flow_run UNSIGNED_LONG NOT NULL, "
+ + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
+ + "entity_id VARCHAR NOT NULL, "
+ + "timestamp UNSIGNED_LONG NOT NULL, event_id VARCHAR NOT NULL, "
+ + EVENT_INFO_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY "
+ + "CONSTRAINT pk PRIMARY KEY("
+ + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
+ + "type, entity_id, timestamp DESC, event_id))";
+ stmt.executeUpdate(sql);
+ sql = "CREATE TABLE IF NOT EXISTS " + METRIC_TABLE_NAME
+ + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
+ + "flow_run UNSIGNED_LONG NOT NULL, "
+ + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
+ + "entity_id VARCHAR NOT NULL, "
+ + "metric_id VARCHAR NOT NULL, "
+ + "singledata VARBINARY, "
+ + "time UNSIGNED_LONG "
+ + "CONSTRAINT pk PRIMARY KEY("
+ + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
+ + "type, entity_id, metric_id))";
+ stmt.executeUpdate(sql);
+ conn.commit();
+ } catch (SQLException se) {
+ LOG.error("Failed in init data " + se.getLocalizedMessage());
+ throw se;
+ }
+ return;
+ }
+
+ private static class DynamicColumns<K> {
+ static final String COLUMN_FAMILY_TYPE_BYTES = " VARBINARY";
+ static final String COLUMN_FAMILY_TYPE_STRING = " VARCHAR";
+ String columnFamilyPrefix;
+ String type;
+ Set<K> columns;
+
+ public DynamicColumns(String columnFamilyPrefix, String type,
+ Set<K> keyValues) {
+ this.columnFamilyPrefix = columnFamilyPrefix;
+ this.columns = keyValues;
+ this.type = type;
+ }
+ }
+
+ private static <K> StringBuilder appendColumnsSQL(
+ StringBuilder colNames, DynamicColumns<K> cfInfo) {
+ // Prepare the sql template by iterating through all keys
+ for (K key : cfInfo.columns) {
+ colNames.append(",").append(cfInfo.columnFamilyPrefix)
+ .append(key.toString()).append(cfInfo.type);
+ }
+ return colNames;
+ }
+
+ private static <K, V> int setValuesForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos,
+ boolean converToBytes) throws SQLException {
+ int idx = startPos;
+ for (Map.Entry<K, V> entry : keyValues.entrySet()) {
+ V value = entry.getValue();
+ if (value instanceof Collection) {
+ ps.setString(idx++, StringUtils.join(
+ (Collection) value, PHOENIX_STORAGE_SEPARATOR));
+ } else {
+ if (converToBytes) {
+ try {
+ ps.setBytes(idx++, GenericObjectMapper.write(entry.getValue()));
+ } catch (IOException ie) {
+ LOG.error("Exception in converting values into bytes "
+ + ie.getMessage());
+ throw new SQLException(ie);
+ }
+ } else {
+ ps.setString(idx++, value.toString());
+ }
+ }
+ }
+ return idx;
+ }
+
+ private static <K, V> int setBytesForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos)
+ throws SQLException {
+ return setValuesForColumnFamily(ps, keyValues, startPos, true);
+ }
+
+ private static <K, V> int setStringsForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos)
+ throws SQLException {
+ return setValuesForColumnFamily(ps, keyValues, startPos, false);
+ }
+
+ private static int setStringsForPrimaryKey(PreparedStatement ps,
+ TimelineCollectorContext context, TimelineEntity entity, int startPos)
+ throws SQLException {
+ int idx = startPos;
+ ps.setString(idx++, context.getClusterId());
+ ps.setString(idx++, context.getUserId());
+ ps.setString(idx++,
+ context.getFlowName());
+ ps.setString(idx++, context.getFlowVersion());
+ ps.setLong(idx++, context.getFlowRunId());
+ ps.setString(idx++, context.getAppId());
+ ps.setString(idx++, entity.getType());
+ ps.setString(idx++, entity.getId());
+ return idx;
+ }
+
+ private static void storeEntityVariableLengthFields(TimelineEntity entity,
+ TimelineCollectorContext context, Connection conn) throws SQLException {
+ int numPlaceholders = 0;
+ StringBuilder columnDefs = new StringBuilder(
+ StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
+ if (entity.getConfigs() != null) {
+ Set<String> keySet = entity.getConfigs().keySet();
+ appendColumnsSQL(columnDefs, new DynamicColumns<>(
+ CONFIG_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
+ keySet));
+ numPlaceholders += keySet.size();
+ }
+ if (entity.getInfo() != null) {
+ Set<String> keySet = entity.getInfo().keySet();
+ appendColumnsSQL(columnDefs, new DynamicColumns<>(
+ INFO_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_BYTES,
+ keySet));
+ numPlaceholders += keySet.size();
+ }
+ if (entity.getIsRelatedToEntities() != null) {
+ Set<String> keySet = entity.getIsRelatedToEntities().keySet();
+ appendColumnsSQL(columnDefs, new DynamicColumns<>(
+ IS_RELATED_TO_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
+ keySet));
+ numPlaceholders += keySet.size();
+ }
+ if (entity.getRelatesToEntities() != null) {
+ Set<String> keySet = entity.getRelatesToEntities().keySet();
+ appendColumnsSQL(columnDefs, new DynamicColumns<>(
+ RELATES_TO_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
+ keySet));
+ numPlaceholders += keySet.size();
+ }
+ if (numPlaceholders == 0) {
+ return;
+ }
+ StringBuilder placeholders = new StringBuilder();
+ placeholders.append(
+ StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length));
+ // numPlaceholders >= 1 now
+ placeholders.append("?")
+ .append(StringUtils.repeat(",?", numPlaceholders - 1));
+ String sqlVariableLengthFields = new StringBuilder("UPSERT INTO ")
+ .append(ENTITY_TABLE_NAME).append(" (").append(columnDefs)
+ .append(") VALUES(").append(placeholders).append(")").toString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL statement for variable length fields: "
+ + sqlVariableLengthFields);
+ }
+ // Use try with resource statement for the prepared statement
+ try (PreparedStatement psVariableLengthFields =
+ conn.prepareStatement(sqlVariableLengthFields)) {
+ int idx = setStringsForPrimaryKey(
+ psVariableLengthFields, context, entity, 1);
+ if (entity.getConfigs() != null) {
+ idx = setStringsForColumnFamily(
+ psVariableLengthFields, entity.getConfigs(), idx);
+ }
+ if (entity.getInfo() != null) {
+ idx = setBytesForColumnFamily(
+ psVariableLengthFields, entity.getInfo(), idx);
+ }
+ if (entity.getIsRelatedToEntities() != null) {
+ idx = setStringsForColumnFamily(
+ psVariableLengthFields, entity.getIsRelatedToEntities(), idx);
+ }
+ if (entity.getRelatesToEntities() != null) {
+ idx = setStringsForColumnFamily(
+ psVariableLengthFields, entity.getRelatesToEntities(), idx);
+ }
+ psVariableLengthFields.execute();
+ }
+ }
+
+ private static void storeMetrics(TimelineEntity entity,
+ TimelineCollectorContext context, Connection conn) throws SQLException {
+ if (entity.getMetrics() == null) {
+ return;
+ }
+ Set<TimelineMetric> metrics = entity.getMetrics();
+ for (TimelineMetric metric : metrics) {
+ StringBuilder sqlColumns = new StringBuilder(
+ StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
+ sqlColumns.append(",")
+ .append(StringUtils.join(TIMELINE_METRIC_EXTRA_PK_LIST, ","));
+ sqlColumns.append(",").append("singledata, time");
+ StringBuilder placeholders = new StringBuilder();
+ placeholders.append(
+ StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length))
+ .append(StringUtils.repeat("?,", TIMELINE_METRIC_EXTRA_PK_LIST.length));
+ placeholders.append("?, ?");
+ String sqlMetric = new StringBuilder("UPSERT INTO ")
+ .append(METRIC_TABLE_NAME).append(" (").append(sqlColumns)
+ .append(") VALUES(").append(placeholders).append(")").toString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL statement for metric: " + sqlMetric);
+ }
+ try (PreparedStatement psMetrics = conn.prepareStatement(sqlMetric)) {
+ if (metric.getType().equals(TimelineMetric.Type.TIME_SERIES)) {
+ LOG.warn("The incoming timeline metric contains time series data, "
+ + "which is currently not supported by Phoenix storage. "
+ + "Time series will be truncated. ");
+ }
+ int idx = setStringsForPrimaryKey(psMetrics, context, entity, 1);
+ psMetrics.setString(idx++, metric.getId());
+ Iterator<Map.Entry<Long, Number>> currNumIter =
+ metric.getValues().entrySet().iterator();
+ if (currNumIter.hasNext()) {
+ // TODO: support time series storage
+ Map.Entry<Long, Number> currEntry = currNumIter.next();
+ psMetrics.setBytes(idx++,
+ GenericObjectMapper.write(currEntry.getValue()));
+ psMetrics.setLong(idx++, currEntry.getKey());
+ } else {
+ psMetrics.setBytes(idx++, GenericObjectMapper.write(null));
+ LOG.warn("The incoming metric contains an empty value set. ");
+ }
+ psMetrics.execute();
+ } catch (IOException ie) {
+ LOG.error("Exception on converting single data to bytes: "
+ + ie.getMessage());
+ throw new SQLException(ie);
+ }
+ }
+ }
+
+ private static void storeEvents(TimelineEntity entity,
+ TimelineCollectorContext context, Connection conn) throws SQLException {
+ if (entity.getEvents() == null) {
+ return;
+ }
+ Set<TimelineEvent> events = entity.getEvents();
+ for (TimelineEvent event : events) {
+ // We need this number to check if the incoming event's info field is empty
+ int numPlaceholders = 0;
+ StringBuilder sqlColumns = new StringBuilder(
+ StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
+ sqlColumns.append(",")
+ .append(StringUtils.join(TIMELINE_EVENT_EXTRA_PK_LIST, ","));
+ appendColumnsSQL(sqlColumns, new DynamicColumns<>(
+ EVENT_INFO_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_BYTES,
+ event.getInfo().keySet()));
+ numPlaceholders += event.getInfo().keySet().size();
+ if (numPlaceholders == 0) {
+ continue;
+ }
+ StringBuilder placeholders = new StringBuilder();
+ placeholders.append(
+ StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length))
+ .append(StringUtils.repeat("?,", TIMELINE_EVENT_EXTRA_PK_LIST.length));
+ // numPlaceholders >= 1 now
+ placeholders.append("?")
+ .append(StringUtils.repeat(",?", numPlaceholders - 1));
+ String sqlEvents = new StringBuilder("UPSERT INTO ")
+ .append(EVENT_TABLE_NAME).append(" (").append(sqlColumns)
+ .append(") VALUES(").append(placeholders).append(")").toString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL statement for events: " + sqlEvents);
+ }
+ try (PreparedStatement psEvent = conn.prepareStatement(sqlEvents)) {
+ int idx = setStringsForPrimaryKey(psEvent, context, entity, 1);
+ psEvent.setLong(idx++, event.getTimestamp());
+ psEvent.setString(idx++, event.getId());
+ setBytesForColumnFamily(psEvent, event.getInfo(), idx);
+ psEvent.execute();
+ }
+ }
+ }
+
+ // WARNING: This method will permanently drop a table!
+ @Private
+ @VisibleForTesting
+ void dropTable(String tableName) throws Exception {
+ try (Connection conn = getConnection();
+ Statement stmt = conn.createStatement()) {
+ String sql = "DROP TABLE " + tableName;
+ stmt.executeUpdate(sql);
+ } catch (SQLException se) {
+ LOG.error("Failed in dropping entity table " + se.getLocalizedMessage());
+ throw se;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d103fbe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
new file mode 100644
index 0000000..a55893e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+public class TestPhoenixTimelineWriterImpl {
+ private PhoenixTimelineWriterImpl writer;
+
+ @Before
+ public void setup() throws Exception {
+ // TODO: launch a miniphoenix cluster, or else we're directly operating on
+ // the active Phoenix cluster
+ YarnConfiguration conf = new YarnConfiguration();
+ writer = createPhoenixWriter(conf);
+ }
+
+ @Ignore
+ @Test
+ public void testPhoenixWriterBasic() throws Exception {
+ // Set up a list of timeline entities and write them back to Phoenix
+ int numEntity = 12;
+ TimelineEntities te =
+ TestTimelineWriterImpl.getStandardTestTimelineEntities(numEntity);
+ writer.write("cluster_1", "user1", "testFlow", "version1", 1l, "app_test_1", te);
+ // Verify if we're storing all entities
+ String sql = "SELECT COUNT(entity_id) FROM "
+ + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME;
+ verifySQLWithCount(sql, numEntity, "Number of entities should be ");
+ // Check config (half of all entities)
+ sql = "SELECT COUNT(c.config) FROM "
+ + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME + "(c.config VARCHAR) ";
+ verifySQLWithCount(sql, (numEntity / 2),
+ "Number of entities with config should be ");
+ // Check info (half of all entities)
+ sql = "SELECT COUNT(i.info1) FROM "
+ + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME + "(i.info1 VARBINARY) ";
+ verifySQLWithCount(sql, (numEntity / 2),
+ "Number of entities with info should be ");
+ // Check config and info (a quarter of all entities)
+ sql = "SELECT COUNT(entity_id) FROM "
+ + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME
+ + "(c.config VARCHAR, i.info1 VARBINARY) "
+ + "WHERE c.config IS NOT NULL AND i.info1 IS NOT NULL";
+ verifySQLWithCount(sql, (numEntity / 4),
+ "Number of entities with both config and info should be ");
+ // Check relatesToEntities and isRelatedToEntities
+ sql = "SELECT COUNT(entity_id) FROM "
+ + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME
+ + "(rt.testType VARCHAR, ir.testType VARCHAR) "
+ + "WHERE rt.testType IS NOT NULL AND ir.testType IS NOT NULL";
+ verifySQLWithCount(sql, numEntity - 2,
+ "Number of entities with both relatesTo and isRelatedTo should be ");
+ // Check event
+ sql = "SELECT COUNT(entity_id) FROM "
+ + PhoenixTimelineWriterImpl.EVENT_TABLE_NAME;
+ verifySQLWithCount(sql, (numEntity / 4), "Number of events should be ");
+ // Check metrics
+ sql = "SELECT COUNT(entity_id) FROM "
+ + PhoenixTimelineWriterImpl.METRIC_TABLE_NAME;
+ verifySQLWithCount(sql, (numEntity / 4), "Number of events should be ");
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ // Note: it is assumed that we're working on a test only cluster, or else
+ // this cleanup process will drop the entity table.
+ writer.dropTable(PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME);
+ writer.dropTable(PhoenixTimelineWriterImpl.EVENT_TABLE_NAME);
+ writer.dropTable(PhoenixTimelineWriterImpl.METRIC_TABLE_NAME);
+ writer.serviceStop();
+ }
+
+ private static PhoenixTimelineWriterImpl createPhoenixWriter(
+ YarnConfiguration conf) throws Exception{
+ PhoenixTimelineWriterImpl myWriter = new PhoenixTimelineWriterImpl();
+ myWriter.serviceInit(conf);
+ return myWriter;
+ }
+
+ private void verifySQLWithCount(String sql, int targetCount, String message)
+ throws Exception{
+ try (
+ Statement stmt =
+ PhoenixTimelineWriterImpl.getConnection().createStatement();
+ ResultSet rs = stmt.executeQuery(sql)) {
+ assertTrue("Result set empty on statement " + sql, rs.next());
+ assertNotNull("Fail to execute query " + sql, rs);
+ assertEquals(message + " " + targetCount, targetCount, rs.getInt(1));
+ } catch (SQLException se) {
+ fail("SQL exception on query: " + sql
+ + " With exception message: " + se.getLocalizedMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d103fbe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
new file mode 100644
index 0000000..7a7afc0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+
+public class TestTimelineWriterImpl {
+ static TimelineEntities getStandardTestTimelineEntities(int listSize) {
+ TimelineEntities te = new TimelineEntities();
+ for (int i = 0; i < listSize; i++) {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "hello" + i;
+ String type = "testType";
+ entity.setId(id);
+ entity.setType(type);
+ entity.setCreatedTime(1425016501000L + i);
+ entity.setModifiedTime(1425016502000L + i);
+ if (i > 0) {
+ entity.addRelatesToEntity(type, "hello" + i);
+ entity.addRelatesToEntity(type, "hello" + (i - 1));
+ }
+ if (i < listSize - 1) {
+ entity.addIsRelatedToEntity(type, "hello" + i);
+ entity.addIsRelatedToEntity(type, "hello" + (i + 1));
+ }
+ int category = i % 4;
+ switch (category) {
+ case 0:
+ entity.addConfig("config", "config" + i);
+ // Fall through deliberately
+ case 1:
+ entity.addInfo("info1", new Integer(i));
+ entity.addInfo("info2", "helloworld");
+ // Fall through deliberately
+ case 2:
+ break;
+ case 3:
+ entity.addConfig("config", "config" + i);
+ TimelineEvent event = new TimelineEvent();
+ event.setId("test event");
+ event.setTimestamp(1425016501100L + i);
+ event.addInfo("test_info", "content for " + entity.getId());
+ event.addInfo("test_info1", new Integer(i));
+ entity.addEvent(event);
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId("HDFS_BYTES_READ");
+ metric.addValue(1425016501100L + i, 8000 + i);
+ entity.addMetric(metric);
+ break;
+ }
+ te.addEntity(entity);
+ }
+ return te;
+ }
+}
[03/50] [abbrv] hadoop git commit: YARN-3551. Consolidate data model
change according to the backend implementation (Zhijie Shen via sale)
Posted by sj...@apache.org.
YARN-3551. Consolidate data model change according to the backend implementation (Zhijie Shen via sale)
(cherry picked from commit 557a3950bddc837469244835f5577899080115d8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27d67784
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27d67784
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27d67784
Branch: refs/heads/YARN-2928-rebase
Commit: 27d67784f53648d36a680bacffd50e4843d47a9b
Parents: 628e9c1
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon May 4 16:10:20 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:39:30 2015 -0700
----------------------------------------------------------------------
.../mapred/TimelineServicePerformanceV2.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/timelineservice/TimelineEntity.java | 16 +--
.../records/timelineservice/TimelineMetric.java | 131 +++++++++----------
.../TestTimelineServiceRecords.java | 81 +++++++++---
.../monitor/ContainersMonitorImpl.java | 4 +-
.../TestTimelineServiceClientIntegration.java | 6 +
7 files changed, 146 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d67784/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
index de46617..1c2e28d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
@@ -261,7 +261,7 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
// add a metric
TimelineMetric metric = new TimelineMetric();
metric.setId("foo_metric");
- metric.setSingleData(123456789L);
+ metric.addValue(System.currentTimeMillis(), 123456789L);
entity.addMetric(metric);
// add a config
entity.addConfig("foo", "bar");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d67784/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1ac7c20..8a77216 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -58,6 +58,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3431. Sub resources of timeline entity needs to be passed to a separate
endpoint. (Zhijie Shen via junping_du)
+ YARN-3551. Consolidate data model change according to the backend
+ implementation (Zhijie Shen via sjlee)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d67784/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 6cab753..3be7f52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -80,7 +80,7 @@ public class TimelineEntity {
private TimelineEntity real;
private Identifier identifier;
private HashMap<String, Object> info = new HashMap<>();
- private HashMap<String, Object> configs = new HashMap<>();
+ private HashMap<String, String> configs = new HashMap<>();
private Set<TimelineMetric> metrics = new HashSet<>();
private Set<TimelineEvent> events = new HashSet<>();
private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>();
@@ -213,7 +213,7 @@ public class TimelineEntity {
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "configs")
- public HashMap<String, Object> getConfigsJAXB() {
+ public HashMap<String, String> getConfigsJAXB() {
if (real == null) {
return configs;
} else {
@@ -221,7 +221,7 @@ public class TimelineEntity {
}
}
- public Map<String, Object> getConfigs() {
+ public Map<String, String> getConfigs() {
if (real == null) {
return configs;
} else {
@@ -229,19 +229,19 @@ public class TimelineEntity {
}
}
- public void setConfigs(Map<String, Object> configs) {
+ public void setConfigs(Map<String, String> configs) {
if (real == null) {
if (configs != null && !(configs instanceof HashMap)) {
- this.configs = new HashMap<String, Object>(configs);
+ this.configs = new HashMap<String, String>(configs);
} else {
- this.configs = (HashMap<String, Object>) configs;
+ this.configs = (HashMap<String, String>) configs;
}
} else {
real.setConfigs(configs);
}
}
- public void addConfigs(Map<String, Object> configs) {
+ public void addConfigs(Map<String, String> configs) {
if (real == null) {
this.configs.putAll(configs);
} else {
@@ -249,7 +249,7 @@ public class TimelineEntity {
}
}
- public void addConfig(String key, Object value) {
+ public void addConfig(String key, String value) {
if (real == null) {
configs.put(key, value);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d67784/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index 57babbf..c897d39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -24,113 +24,104 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
+import java.util.Comparator;
import java.util.Map;
+import java.util.TreeMap;
@XmlRootElement(name = "metric")
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class TimelineMetric {
- private String id;
- private HashMap<String, Object> info = new HashMap<>();
- private Object singleData;
- private HashMap<Long, Object> timeSeries = new LinkedHashMap<>();
- private long startTime;
- private long endTime;
-
- public TimelineMetric() {
+ public static enum Type {
+ SINGLE_VALUE,
+ TIME_SERIES
}
- @XmlElement(name = "id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
+ private Type type;
+ private String id;
+ private Comparator<Long> reverseComparator = new Comparator<Long>() {
+ @Override
+ public int compare(Long l1, Long l2) {
+ return -l1.compareTo(l2);
+ }
+ };
+ private TreeMap<Long, Number> values = new TreeMap<>(reverseComparator);
- // required by JAXB
- @InterfaceAudience.Private
- @XmlElement(name = "info")
- public HashMap<String, Object> getInfoJAXB() {
- return info;
+ public TimelineMetric() {
+ this(Type.SINGLE_VALUE);
}
- public Map<String, Object> getInfo() {
- return info;
+ public TimelineMetric(Type type) {
+ this.type = type;
}
- public void setInfo(Map<String, Object> info) {
- if (info != null && !(info instanceof HashMap)) {
- this.info = new HashMap<String, Object>(info);
- } else {
- this.info = (HashMap<String, Object>) info;
- }
- }
- public void addInfo(Map<String, Object> info) {
- this.info.putAll(info);
+ @XmlElement(name = "type")
+ public Type getType() {
+ return type;
}
- public void addInfo(String key, Object value) {
- info.put(key, value);
+ public void setType(Type type) {
+ this.type = type;
}
- @XmlElement(name = "data")
- public Object getSingleData() {
- return singleData;
+ @XmlElement(name = "id")
+ public String getId() {
+ return id;
}
- public void setSingleData(Object singleData) {
- this.singleData = singleData;
+ public void setId(String id) {
+ this.id = id;
}
// required by JAXB
@InterfaceAudience.Private
- @XmlElement(name = "timeseries")
- public HashMap<Long, Object> getTimeSeriesJAXB() {
- return timeSeries;
+ @XmlElement(name = "values")
+ public TreeMap<Long, Number> getValuesJAXB() {
+ return values;
}
- public Map<Long, Object> getTimeSeries() {
- return timeSeries;
+ public Map<Long, Number> getValues() {
+ return values;
}
- public void setTimeSeries(Map<Long, Object> timeSeries) {
- if (timeSeries != null && !(timeSeries instanceof LinkedHashMap)) {
- this.timeSeries = new LinkedHashMap<Long, Object>(timeSeries);
+ public void setValues(Map<Long, Number> values) {
+ if (type == Type.SINGLE_VALUE) {
+ overwrite(values);
} else {
- this.timeSeries = (LinkedHashMap<Long, Object>) timeSeries;
+ if (values != null) {
+ this.values = new TreeMap<Long, Number>(reverseComparator);
+ this.values.putAll(values);
+ } else {
+ this.values = null;
+ }
}
}
- public void addTimeSeries(Map<Long, Object> timeSeries) {
- this.timeSeries.putAll(timeSeries);
- }
-
- public void addTimeSeriesData(long timestamp, Object value) {
- timeSeries.put(timestamp, value);
- }
-
- @XmlElement(name = "starttime")
- public long getStartTime() {
- return startTime;
- }
-
- public void setStartTime(long startTime) {
- this.startTime = startTime;
+ public void addValues(Map<Long, Number> values) {
+ if (type == Type.SINGLE_VALUE) {
+ overwrite(values);
+ } else {
+ this.values.putAll(values);
+ }
}
- @XmlElement(name = "endtime")
- public long getEndTime() {
- return endTime;
+ public void addValue(long timestamp, Number value) {
+ if (type == Type.SINGLE_VALUE) {
+ values.clear();
+ }
+ values.put(timestamp, value);
}
- public void setEndTime(long endTime) {
- this.endTime = endTime;
+ private void overwrite(Map<Long, Number> values) {
+ if (values.size() > 1) {
+ throw new IllegalArgumentException(
+ "Values cannot contain more than one point in " +
+ Type.SINGLE_VALUE + " mode");
+ }
+ this.values.clear();
+ this.values.putAll(values);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d67784/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index caa3f3f..95b922e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -28,7 +28,11 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Test;
import org.junit.Assert;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
public class TestTimelineServiceRecords {
@@ -41,38 +45,83 @@ public class TestTimelineServiceRecords {
entity.setType("test type 1");
entity.setId("test id 1");
entity.addInfo("test info key 1", "test info value 1");
- entity.addInfo("test info key 2", "test info value 2");
+ entity.addInfo("test info key 2",
+ Arrays.asList("test info value 2", "test info value 3"));
+ entity.addInfo("test info key 3", true);
+ Assert.assertTrue(
+ entity.getInfo().get("test info key 3") instanceof Boolean);
entity.addConfig("test config key 1", "test config value 1");
entity.addConfig("test config key 2", "test config value 2");
- TimelineMetric metric1 = new TimelineMetric();
+
+ TimelineMetric metric1 =
+ new TimelineMetric(TimelineMetric.Type.TIME_SERIES);
metric1.setId("test metric id 1");
- metric1.addInfo("test info key 1", "test info value 1");
- metric1.addInfo("test info key 2", "test info value 2");
- metric1.addTimeSeriesData(1L, "test time series 1");
- metric1.addTimeSeriesData(2L, "test time series 2");
- metric1.setStartTime(0L);
- metric1.setEndTime(1L);
+ metric1.addValue(1L, 1.0F);
+ metric1.addValue(3L, 3.0D);
+ metric1.addValue(2L, 2);
+ Assert.assertEquals(TimelineMetric.Type.TIME_SERIES, metric1.getType());
+ Iterator<Map.Entry<Long, Number>> itr =
+ metric1.getValues().entrySet().iterator();
+ Map.Entry<Long, Number> entry = itr.next();
+ Assert.assertEquals(new Long(3L), entry.getKey());
+ Assert.assertEquals(new Double(3.0D), entry.getValue());
+ entry = itr.next();
+ Assert.assertEquals(new Long(2L), entry.getKey());
+ Assert.assertEquals(new Integer(2), entry.getValue());
+ entry = itr.next();
+ Assert.assertEquals(new Long(1L), entry.getKey());
+ Assert.assertEquals(new Float(1.0F), entry.getValue());
+ Assert.assertFalse(itr.hasNext());
entity.addMetric(metric1);
- TimelineMetric metric2 = new TimelineMetric();
+
+ TimelineMetric metric2 =
+ new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
metric2.setId("test metric id 1");
- metric2.addInfo("test info key 1", "test info value 1");
- metric2.addInfo("test info key 2", "test info value 2");
- metric2.setSingleData("test info value 3");
- metric1.setStartTime(0L);
- metric1.setEndTime(1L);
+ metric2.addValue(3L, (short) 3);
+ Assert.assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric2.getType());
+ Assert.assertTrue(
+ metric2.getValues().values().iterator().next() instanceof Short);
+ Map<Long, Number> points = new HashMap<>();
+ points.put(4L, 4.0D);
+ points.put(5L, 5.0D);
+ try {
+ metric2.setValues(points);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Values cannot contain more than one point in"));
+ }
+ try {
+ metric2.addValues(points);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Values cannot contain more than one point in"));
+ }
entity.addMetric(metric2);
+
TimelineEvent event1 = new TimelineEvent();
event1.setId("test event id 1");
event1.addInfo("test info key 1", "test info value 1");
- event1.addInfo("test info key 2", "test info value 2");
+ event1.addInfo("test info key 2",
+ Arrays.asList("test info value 2", "test info value 3"));
+ event1.addInfo("test info key 3", true);
+ Assert.assertTrue(
+ event1.getInfo().get("test info key 3") instanceof Boolean);
event1.setTimestamp(0L);
entity.addEvent(event1);
+
TimelineEvent event2 = new TimelineEvent();
event2.setId("test event id 2");
event2.addInfo("test info key 1", "test info value 1");
- event2.addInfo("test info key 2", "test info value 2");
+ event2.addInfo("test info key 2",
+ Arrays.asList("test info value 2", "test info value 3"));
+ event2.addInfo("test info key 3", true);
+ Assert.assertTrue(
+ event2.getInfo().get("test info key 3") instanceof Boolean);
event2.setTimestamp(1L);
entity.addEvent(event2);
+
entity.setCreatedTime(0L);
entity.setModifiedTime(1L);
entity.addRelatesToEntity("test type 2", "test id 2");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d67784/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index ecec151..6f08990 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -559,7 +559,7 @@ public class ContainersMonitorImpl extends AbstractService implements
ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric memoryMetric = new TimelineMetric();
memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
- memoryMetric.addTimeSeriesData(currentTime, currentPmemUsage);
+ memoryMetric.addValue(currentTime, currentPmemUsage);
entity.addMetric(memoryMetric);
}
// if cpuUsageTotalCoresPercentage data is available
@@ -567,7 +567,7 @@ public class ContainersMonitorImpl extends AbstractService implements
ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric cpuMetric = new TimelineMetric();
cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
- cpuMetric.addTimeSeriesData(currentTime,
+ cpuMetric.addValue(currentTime,
cpuUsageTotalCoresPercentage);
entity.addMetric(cpuMetric);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d67784/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 044148a..69031a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -81,6 +81,12 @@ public class TestTimelineServiceClientIntegration {
TimelineEntity entity = new TimelineEntity();
entity.setType("test entity type");
entity.setId("test entity id");
+ TimelineMetric metric =
+ new TimelineMetric(TimelineMetric.Type.TIME_SERIES);
+ metric.setId("test metric id");
+ metric.addValue(1L, 1.0D);
+ metric.addValue(2L, 2.0D);
+ entity.addMetric(metric);
client.putEntities(entity);
client.putEntitiesAsync(entity);
} finally {
[39/50] [abbrv] hadoop git commit: YARN-3901. Populate flow run data
in the flow_run & flow activity tables (Vrushali C via sjlee)
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
new file mode 100644
index 0000000..b4a0c74
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -0,0 +1,372 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests the FlowRun and FlowActivity Tables
+ */
+public class TestHBaseStorageFlowActivity {
+
+ private static HBaseTestingUtility util;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ util = new HBaseTestingUtility();
+ Configuration conf = util.getConfiguration();
+ conf.setInt("hfile.format.version", 3);
+ util.startMiniCluster();
+ createSchema();
+ }
+
+ private static void createSchema() throws IOException {
+ TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
+ }
+
+ /**
+ * Writes 4 timeline entities belonging to one flow run through the
+ * {@link HBaseTimelineWriterImpl}
+ *
+ * Checks the flow run table contents
+ *
+ * The first entity has a created event, metrics and a finish event.
+ *
+ * The second entity has a created event and this is the entity with smallest
+ * start time. This should be the start time for the flow run.
+ *
+ * The third entity has a finish event and this is the entity with the max end
+ * time. This should be the end time for the flow run.
+ *
+ * The fourth entity has a created event which has a start time that is
+ * greater than min start time.
+ *
+ * The test also checks in the flow activity table that one entry has been
+ * made for all of these 4 application entities since they belong to the same
+ * flow run.
+ */
+ @Test
+ public void testWriteFlowRunMinMax() throws Exception {
+
+ TimelineEntities te = new TimelineEntities();
+ te.addEntity(TestFlowDataGenerator.getEntity1());
+
+ HBaseTimelineWriterImpl hbi = null;
+ Configuration c1 = util.getConfiguration();
+ String cluster = "testWriteFlowRunMinMaxToHBase_cluster1";
+ String user = "testWriteFlowRunMinMaxToHBase_user1";
+ String flow = "testing_flowRun_flow_name";
+ String flowVersion = "CF7022C10F1354";
+ Long runid = 1002345678919L;
+ String appName = "application_100000000000_1111";
+ long endTs = 1439750690000L;
+ TimelineEntity entityMinStartTime = TestFlowDataGenerator
+ .getEntityMinStartTime();
+
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+
+ // write another entity with the right min start time
+ te = new TimelineEntities();
+ te.addEntity(entityMinStartTime);
+ appName = "application_100000000000_3333";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+
+ // writer another entity for max end time
+ TimelineEntity entityMaxEndTime = TestFlowDataGenerator
+ .getEntityMaxEndTime(endTs);
+ te = new TimelineEntities();
+ te.addEntity(entityMaxEndTime);
+ appName = "application_100000000000_4444";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+
+ // writer another entity with greater start time
+ TimelineEntity entityGreaterStartTime = TestFlowDataGenerator
+ .getEntityGreaterStartTime();
+ te = new TimelineEntities();
+ te.addEntity(entityGreaterStartTime);
+ appName = "application_1000000000000000_2222";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+
+ // flush everything to hbase
+ hbi.flush();
+ } finally {
+ hbi.close();
+ }
+
+ Connection conn = ConnectionFactory.createConnection(c1);
+ // check in flow activity table
+ Table table1 = conn.getTable(TableName
+ .valueOf(FlowActivityTable.DEFAULT_TABLE_NAME));
+ byte[] startRow = FlowActivityRowKey.getRowKey(cluster, user, flow);
+ Get g = new Get(startRow);
+ Result r1 = table1.get(g);
+ assertNotNull(r1);
+ assertTrue(!r1.isEmpty());
+ Map<byte[], byte[]> values = r1.getFamilyMap(FlowActivityColumnFamily.INFO
+ .getBytes());
+ assertEquals(1, values.size());
+ byte[] row = r1.getRow();
+ FlowActivityRowKey flowActivityRowKey = FlowActivityRowKey.parseRowKey(row);
+ assertNotNull(flowActivityRowKey);
+ assertEquals(cluster, flowActivityRowKey.getClusterId());
+ assertEquals(user, flowActivityRowKey.getUserId());
+ assertEquals(flow, flowActivityRowKey.getFlowId());
+ long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
+ .currentTimeMillis());
+ assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
+ assertEquals(1, values.size());
+ checkFlowActivityRunId(runid, flowVersion, values);
+ }
+
+ /**
+ * Write 1 application entity and checks the record for today in the flow
+ * activity table
+ */
+ @Test
+ public void testWriteFlowActivityOneFlow() throws Exception {
+ String cluster = "testWriteFlowActivityOneFlow_cluster1";
+ String user = "testWriteFlowActivityOneFlow_user1";
+ String flow = "flow_activity_test_flow_name";
+ String flowVersion = "A122110F135BC4";
+ Long runid = 1001111178919L;
+
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entityApp1 = TestFlowDataGenerator.getFlowApp1();
+ te.addEntity(entityApp1);
+
+ HBaseTimelineWriterImpl hbi = null;
+ Configuration c1 = util.getConfiguration();
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ String appName = "application_1111999999_1234";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+ hbi.flush();
+ } finally {
+ hbi.close();
+ }
+ // check flow activity
+ checkFlowActivityTable(cluster, user, flow, flowVersion, runid, c1);
+ }
+
+ private void checkFlowActivityTable(String cluster, String user, String flow,
+ String flowVersion, Long runid, Configuration c1) throws IOException {
+ Scan s = new Scan();
+ s.addFamily(FlowActivityColumnFamily.INFO.getBytes());
+ byte[] startRow = FlowActivityRowKey.getRowKey(cluster, user, flow);
+ s.setStartRow(startRow);
+ String clusterStop = cluster + "1";
+ byte[] stopRow = FlowActivityRowKey.getRowKey(clusterStop, user, flow);
+ s.setStopRow(stopRow);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Table table1 = conn.getTable(TableName
+ .valueOf(FlowActivityTable.DEFAULT_TABLE_NAME));
+ ResultScanner scanner = table1.getScanner(s);
+ int rowCount = 0;
+ for (Result result : scanner) {
+ assertNotNull(result);
+ assertTrue(!result.isEmpty());
+ Map<byte[], byte[]> values = result
+ .getFamilyMap(FlowActivityColumnFamily.INFO.getBytes());
+ rowCount++;
+ byte[] row = result.getRow();
+ FlowActivityRowKey flowActivityRowKey = FlowActivityRowKey
+ .parseRowKey(row);
+ assertNotNull(flowActivityRowKey);
+ assertEquals(cluster, flowActivityRowKey.getClusterId());
+ assertEquals(user, flowActivityRowKey.getUserId());
+ assertEquals(flow, flowActivityRowKey.getFlowId());
+ long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
+ .currentTimeMillis());
+ assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
+ assertEquals(1, values.size());
+ checkFlowActivityRunId(runid, flowVersion, values);
+ }
+ assertEquals(1, rowCount);
+ }
+
+ /**
+ * Writes 3 applications each with a different run id and version for the same
+ * {cluster, user, flow}
+ *
+ * They should be getting inserted into one record in the flow activity table
+ * with 3 columns, one per run id
+ */
+ @Test
+ public void testFlowActivityTableOneFlowMultipleRunIds() throws IOException {
+ String cluster = "testManyRunsFlowActivity_cluster1";
+ String user = "testManyRunsFlowActivity_c_user1";
+ String flow = "flow_activity_test_flow_name";
+ String flowVersion1 = "A122110F135BC4";
+ Long runid1 = 11111111111L;
+
+ String flowVersion2 = "A12222222222C4";
+ long runid2 = 2222222222222L;
+
+ String flowVersion3 = "A1333333333C4";
+ long runid3 = 3333333333333L;
+
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entityApp1 = TestFlowDataGenerator.getFlowApp1();
+ te.addEntity(entityApp1);
+
+ HBaseTimelineWriterImpl hbi = null;
+ Configuration c1 = util.getConfiguration();
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ String appName = "application_11888888888_1111";
+ hbi.write(cluster, user, flow, flowVersion1, runid1, appName, te);
+
+ // write an application with to this flow but a different runid/ version
+ te = new TimelineEntities();
+ te.addEntity(entityApp1);
+ appName = "application_11888888888_2222";
+ hbi.write(cluster, user, flow, flowVersion2, runid2, appName, te);
+
+ // write an application with to this flow but a different runid/ version
+ te = new TimelineEntities();
+ te.addEntity(entityApp1);
+ appName = "application_11888888888_3333";
+ hbi.write(cluster, user, flow, flowVersion3, runid3, appName, te);
+
+ hbi.flush();
+ } finally {
+ hbi.close();
+ }
+ // check flow activity
+ checkFlowActivityTableSeveralRuns(cluster, user, flow, c1, flowVersion1,
+ runid1, flowVersion2, runid2, flowVersion3, runid3);
+
+ }
+
+ private void checkFlowActivityTableSeveralRuns(String cluster, String user,
+ String flow, Configuration c1, String flowVersion1, Long runid1,
+ String flowVersion2, Long runid2, String flowVersion3, Long runid3)
+ throws IOException {
+ Scan s = new Scan();
+ s.addFamily(FlowActivityColumnFamily.INFO.getBytes());
+ byte[] startRow = FlowActivityRowKey.getRowKey(cluster, user, flow);
+ s.setStartRow(startRow);
+ String clusterStop = cluster + "1";
+ byte[] stopRow = FlowActivityRowKey.getRowKey(clusterStop, user, flow);
+ s.setStopRow(stopRow);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Table table1 = conn.getTable(TableName
+ .valueOf(FlowActivityTable.DEFAULT_TABLE_NAME));
+ ResultScanner scanner = table1.getScanner(s);
+ int rowCount = 0;
+ for (Result result : scanner) {
+ assertNotNull(result);
+ assertTrue(!result.isEmpty());
+ byte[] row = result.getRow();
+ FlowActivityRowKey flowActivityRowKey = FlowActivityRowKey
+ .parseRowKey(row);
+ assertNotNull(flowActivityRowKey);
+ assertEquals(cluster, flowActivityRowKey.getClusterId());
+ assertEquals(user, flowActivityRowKey.getUserId());
+ assertEquals(flow, flowActivityRowKey.getFlowId());
+ long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
+ .currentTimeMillis());
+ assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
+
+ Map<byte[], byte[]> values = result
+ .getFamilyMap(FlowActivityColumnFamily.INFO.getBytes());
+ rowCount++;
+ assertEquals(3, values.size());
+ checkFlowActivityRunId(runid1, flowVersion1, values);
+ checkFlowActivityRunId(runid2, flowVersion2, values);
+ checkFlowActivityRunId(runid3, flowVersion3, values);
+ }
+ // the flow activity table is such that it will insert
+ // into current day's record
+ // hence, if this test runs across the midnight boundary,
+ // it may fail since it would insert into two records
+ // one for each day
+ assertEquals(1, rowCount);
+ }
+
+ private void checkFlowActivityRunId(Long runid, String flowVersion,
+ Map<byte[], byte[]> values) throws IOException {
+ byte[] rq = ColumnHelper.getColumnQualifier(
+ FlowActivityColumnPrefix.RUN_ID.getColumnPrefixBytes(),
+ GenericObjectMapper.write(runid));
+ for (Map.Entry<byte[], byte[]> k : values.entrySet()) {
+ String actualQ = Bytes.toString(k.getKey());
+ if (Bytes.toString(rq).equals(actualQ)) {
+ String actualV = (String) GenericObjectMapper.read(k.getValue());
+ assertEquals(flowVersion, actualV);
+ }
+ }
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ util.shutdownMiniCluster();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
new file mode 100644
index 0000000..bf524ea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests the FlowRun and FlowActivity Tables
+ */
+public class TestHBaseStorageFlowRun {
+
+ private static HBaseTestingUtility util;
+
+ private final String metric1 = "MAP_SLOT_MILLIS";
+ private final String metric2 = "HDFS_BYTES_READ";
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ util = new HBaseTestingUtility();
+ Configuration conf = util.getConfiguration();
+ conf.setInt("hfile.format.version", 3);
+ util.startMiniCluster();
+ createSchema();
+ }
+
+ private static void createSchema() throws IOException {
+ TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
+ }
+
+ /**
+ * Writes 4 timeline entities belonging to one flow run through the
+ * {@link HBaseTimelineWriterImpl}
+ *
+ * Checks the flow run table contents
+ *
+ * The first entity has a created event, metrics and a finish event.
+ *
+ * The second entity has a created event and this is the entity with smallest
+ * start time. This should be the start time for the flow run.
+ *
+ * The third entity has a finish event and this is the entity with the max end
+ * time. This should be the end time for the flow run.
+ *
+ * The fourth entity has a created event which has a start time that is
+ * greater than min start time.
+ *
+ */
+ @Test
+ public void testWriteFlowRunMinMax() throws Exception {
+
+ TimelineEntities te = new TimelineEntities();
+ te.addEntity(TestFlowDataGenerator.getEntity1());
+
+ HBaseTimelineWriterImpl hbi = null;
+ Configuration c1 = util.getConfiguration();
+ String cluster = "testWriteFlowRunMinMaxToHBase_cluster1";
+ String user = "testWriteFlowRunMinMaxToHBase_user1";
+ String flow = "testing_flowRun_flow_name";
+ String flowVersion = "CF7022C10F1354";
+ Long runid = 1002345678919L;
+ String appName = "application_100000000000_1111";
+ long endTs = 1439750690000L;
+ TimelineEntity entityMinStartTime = TestFlowDataGenerator
+ .getEntityMinStartTime();
+
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+
+ // write another entity with the right min start time
+ te = new TimelineEntities();
+ te.addEntity(entityMinStartTime);
+ appName = "application_100000000000_3333";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+
+ // writer another entity for max end time
+ TimelineEntity entityMaxEndTime = TestFlowDataGenerator
+ .getEntityMaxEndTime(endTs);
+ te = new TimelineEntities();
+ te.addEntity(entityMaxEndTime);
+ appName = "application_100000000000_4444";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+
+ // writer another entity with greater start time
+ TimelineEntity entityGreaterStartTime = TestFlowDataGenerator
+ .getEntityGreaterStartTime();
+ te = new TimelineEntities();
+ te.addEntity(entityGreaterStartTime);
+ appName = "application_1000000000000000_2222";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+
+ // flush everything to hbase
+ hbi.flush();
+ } finally {
+ hbi.close();
+ }
+
+ Connection conn = ConnectionFactory.createConnection(c1);
+ // check in flow run table
+ Table table1 = conn.getTable(TableName
+ .valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
+ // scan the table and see that we get back the right min and max
+ // timestamps
+ byte[] startRow = FlowRunRowKey.getRowKey(cluster, user, flow, runid);
+ Get g = new Get(startRow);
+ g.addColumn(FlowRunColumnFamily.INFO.getBytes(),
+ FlowRunColumn.MIN_START_TIME.getColumnQualifierBytes());
+ g.addColumn(FlowRunColumnFamily.INFO.getBytes(),
+ FlowRunColumn.MAX_END_TIME.getColumnQualifierBytes());
+ Result r1 = table1.get(g);
+ assertNotNull(r1);
+ assertTrue(!r1.isEmpty());
+ Map<byte[], byte[]> values = r1.getFamilyMap(FlowRunColumnFamily.INFO
+ .getBytes());
+
+ assertEquals(2, r1.size());
+ Long starttime = (Long) GenericObjectMapper.read(values
+ .get(FlowRunColumn.MIN_START_TIME.getColumnQualifierBytes()));
+ Long expmin = entityMinStartTime.getCreatedTime();
+ assertEquals(expmin, starttime);
+ assertEquals(endTs, GenericObjectMapper.read(values
+ .get(FlowRunColumn.MAX_END_TIME.getColumnQualifierBytes())));
+ }
+
+ boolean isFlowRunRowKeyCorrect(byte[] rowKey, String cluster, String user,
+ String flow, Long runid) {
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, -1);
+ assertTrue(rowKeyComponents.length == 4);
+ assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
+ assertEquals(user, Bytes.toString(rowKeyComponents[1]));
+ assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
+ assertEquals(TimelineWriterUtils.invert(runid),
+ Bytes.toLong(rowKeyComponents[3]));
+ return true;
+ }
+
+ /**
+ * Writes two application entities of the same flow run. Each application has
+ * two metrics: slot millis and hdfs bytes read. Each metric has values at two
+ * timestamps.
+ *
+ * Checks the metric values of the flow in the flow run table. Flow metric
+ * values should be the sum of individual metric values that belong to the
+ * latest timestamp for that metric
+ */
+ @Test
+ public void testWriteFlowRunMetricsOneFlow() throws Exception {
+ String cluster = "testWriteFlowRunMetricsOneFlow_cluster1";
+ String user = "testWriteFlowRunMetricsOneFlow_user1";
+ String flow = "testing_flowRun_metrics_flow_name";
+ String flowVersion = "CF7022C10F1354";
+ Long runid = 1002345678919L;
+
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1();
+ te.addEntity(entityApp1);
+
+ HBaseTimelineWriterImpl hbi = null;
+ Configuration c1 = util.getConfiguration();
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ String appName = "application_11111111111111_1111";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+ // write another application with same metric to this flow
+ te = new TimelineEntities();
+ TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2();
+ te.addEntity(entityApp2);
+ appName = "application_11111111111111_2222";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+ hbi.flush();
+ } finally {
+ hbi.close();
+ }
+
+ // check flow run
+ checkFlowRunTable(cluster, user, flow, runid, c1);
+ }
+
+ private void checkFlowRunTable(String cluster, String user, String flow,
+ long runid, Configuration c1) throws IOException {
+ Scan s = new Scan();
+ s.addFamily(FlowRunColumnFamily.INFO.getBytes());
+ byte[] startRow = FlowRunRowKey.getRowKey(cluster, user, flow, runid);
+ s.setStartRow(startRow);
+ String clusterStop = cluster + "1";
+ byte[] stopRow = FlowRunRowKey.getRowKey(clusterStop, user, flow, runid);
+ s.setStopRow(stopRow);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Table table1 = conn.getTable(TableName
+ .valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
+ ResultScanner scanner = table1.getScanner(s);
+
+ int rowCount = 0;
+ for (Result result : scanner) {
+ assertNotNull(result);
+ assertTrue(!result.isEmpty());
+ Map<byte[], byte[]> values = result.getFamilyMap(FlowRunColumnFamily.INFO
+ .getBytes());
+ rowCount++;
+ // check metric1
+ byte[] q = ColumnHelper.getColumnQualifier(
+ FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), metric1);
+ assertTrue(values.containsKey(q));
+ assertEquals(141, GenericObjectMapper.read(values.get(q)));
+
+ // check metric2
+ assertEquals(2, values.size());
+ q = ColumnHelper.getColumnQualifier(
+ FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), metric2);
+ assertTrue(values.containsKey(q));
+ assertEquals(57, GenericObjectMapper.read(values.get(q)));
+ }
+ assertEquals(1, rowCount);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ util.shutdownMiniCluster();
+ }
+}
[16/50] [abbrv] hadoop git commit: YARN-3706. Generalize native HBase
writer for additional tables (Joep Rottinghuis via sjlee)
Posted by sj...@apache.org.
YARN-3706. Generalize native HBase writer for additional tables (Joep Rottinghuis via sjlee)
(cherry picked from commit 9137aeae0dec83f9eff40d12cae712dfd508c0c5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3599c8e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3599c8e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3599c8e5
Branch: refs/heads/YARN-2928-rebase
Commit: 3599c8e5e4be217bd6185570d7b06c2066688743
Parents: 50eaf2b
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu Jun 18 10:49:20 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:36 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../storage/EntityColumnDetails.java | 110 ------
.../storage/EntityColumnFamily.java | 95 -----
.../storage/HBaseTimelineWriterImpl.java | 114 +++---
.../server/timelineservice/storage/Range.java | 59 ----
.../storage/TimelineEntitySchemaConstants.java | 71 ----
.../storage/TimelineSchemaCreator.java | 134 +-------
.../timelineservice/storage/TimelineWriter.java | 3 +-
.../storage/TimelineWriterUtils.java | 344 -------------------
.../storage/common/BaseTable.java | 118 +++++++
.../common/BufferedMutatorDelegator.java | 73 ++++
.../timelineservice/storage/common/Column.java | 59 ++++
.../storage/common/ColumnFamily.java | 34 ++
.../storage/common/ColumnHelper.java | 247 +++++++++++++
.../storage/common/ColumnPrefix.java | 83 +++++
.../timelineservice/storage/common/Range.java | 59 ++++
.../storage/common/Separator.java | 303 ++++++++++++++++
.../common/TimelineEntitySchemaConstants.java | 68 ++++
.../storage/common/TimelineWriterUtils.java | 127 +++++++
.../storage/common/TypedBufferedMutator.java | 28 ++
.../storage/common/package-info.java | 24 ++
.../storage/entity/EntityColumn.java | 141 ++++++++
.../storage/entity/EntityColumnFamily.java | 65 ++++
.../storage/entity/EntityColumnPrefix.java | 212 ++++++++++++
.../storage/entity/EntityRowKey.java | 93 +++++
.../storage/entity/EntityTable.java | 161 +++++++++
.../storage/entity/package-info.java | 25 ++
.../storage/TestHBaseTimelineWriterImpl.java | 252 ++++++++------
.../storage/common/TestSeparator.java | 129 +++++++
.../storage/common/TestTimelineWriterUtils.java | 29 ++
30 files changed, 2301 insertions(+), 962 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a9e6d85..0dc2058 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -93,6 +93,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
zjshen)
+ YARN-3706. Generalize native HBase writer for additional tables (Joep
+ Rottinghuis via sjlee)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
deleted file mode 100644
index 2894c41..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Contains the Info Column Family details like Column names, types and byte
- * representations for
- * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
- * object that is stored in hbase Also has utility functions for storing each of
- * these to the backend
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-enum EntityColumnDetails {
- ID(EntityColumnFamily.INFO, "id"),
- TYPE(EntityColumnFamily.INFO, "type"),
- CREATED_TIME(EntityColumnFamily.INFO, "created_time"),
- MODIFIED_TIME(EntityColumnFamily.INFO, "modified_time"),
- FLOW_VERSION(EntityColumnFamily.INFO, "flow_version"),
- PREFIX_IS_RELATED_TO(EntityColumnFamily.INFO, "r"),
- PREFIX_RELATES_TO(EntityColumnFamily.INFO, "s"),
- PREFIX_EVENTS(EntityColumnFamily.INFO, "e");
-
- private final EntityColumnFamily columnFamily;
- private final String value;
- private final byte[] inBytes;
-
- private EntityColumnDetails(EntityColumnFamily columnFamily,
- String value) {
- this.columnFamily = columnFamily;
- this.value = value;
- this.inBytes = Bytes.toBytes(this.value.toLowerCase());
- }
-
- public String getValue() {
- return value;
- }
-
- byte[] getInBytes() {
- return inBytes;
- }
-
- void store(byte[] rowKey, BufferedMutator entityTable, Object inputValue)
- throws IOException {
- TimelineWriterUtils.store(rowKey, entityTable,
- this.columnFamily.getInBytes(), null, this.getInBytes(), inputValue,
- null);
- }
-
- /**
- * stores events data with column prefix
- */
- void store(byte[] rowKey, BufferedMutator entityTable, byte[] idBytes,
- String key, Object inputValue) throws IOException {
- TimelineWriterUtils.store(rowKey, entityTable,
- this.columnFamily.getInBytes(),
- // column prefix
- TimelineWriterUtils.join(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
- this.getInBytes(), idBytes),
- // column qualifier
- Bytes.toBytes(key),
- inputValue, null);
- }
-
- /**
- * stores relation entities with a column prefix
- */
- void store(byte[] rowKey, BufferedMutator entityTable, String key,
- Set<String> inputValue) throws IOException {
- TimelineWriterUtils.store(rowKey, entityTable,
- this.columnFamily.getInBytes(),
- // column prefix
- this.getInBytes(),
- // column qualifier
- Bytes.toBytes(key),
- // value
- TimelineWriterUtils.getValueAsString(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR, inputValue),
- // cell timestamp
- null);
- }
-
- // TODO add a method that accepts a byte array,
- // iterates over the enum and returns an enum from those bytes
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
deleted file mode 100644
index e556351..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Contains the Column family names and byte representations for
- * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
- * object that is stored in hbase
- * Also has utility functions for storing each of these to the backend
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-enum EntityColumnFamily {
- INFO("i"),
- CONFIG("c"),
- METRICS("m");
-
- private final String value;
- private final byte[] inBytes;
-
- private EntityColumnFamily(String value) {
- this.value = value;
- this.inBytes = Bytes.toBytes(this.value.toLowerCase());
- }
-
- byte[] getInBytes() {
- return inBytes;
- }
-
- public String getValue() {
- return value;
- }
-
- /**
- * stores the key as column and value as hbase column value in the given
- * column family in the entity table
- *
- * @param rowKey
- * @param entityTable
- * @param inputValue
- * @throws IOException
- */
- public void store(byte[] rowKey, BufferedMutator entityTable, String key,
- String inputValue) throws IOException {
- if (key == null) {
- return;
- }
- TimelineWriterUtils.store(rowKey, entityTable, this.getInBytes(), null,
- Bytes.toBytes(key), inputValue, null);
- }
-
- /**
- * stores the values along with cell timestamp
- *
- * @param rowKey
- * @param entityTable
- * @param key
- * @param timestamp
- * @param inputValue
- * @throws IOException
- */
- public void store(byte[] rowKey, BufferedMutator entityTable, String key,
- Long timestamp, Number inputValue) throws IOException {
- if (key == null) {
- return;
- }
- TimelineWriterUtils.store(rowKey, entityTable, this.getInBytes(), null,
- Bytes.toBytes(key), inputValue, timestamp);
- }
-
- // TODO add a method that accepts a byte array,
- // iterates over the enum and returns an enum from those bytes
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index aa71c6c..e48ca60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -26,19 +26,22 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineEntitySchemaConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
/**
* This implements a hbase based backend for storing application timeline entity
@@ -50,7 +53,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
TimelineWriter {
private Connection conn;
- private BufferedMutator entityTable;
+ private TypedBufferedMutator<EntityTable> entityTable;
private static final Log LOG = LogFactory
.getLog(HBaseTimelineWriterImpl.class);
@@ -72,10 +75,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
super.serviceInit(conf);
Configuration hbaseConf = HBaseConfiguration.create(conf);
conn = ConnectionFactory.createConnection(hbaseConf);
- TableName entityTableName = TableName.valueOf(hbaseConf.get(
- TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
- TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME));
- entityTable = conn.getBufferedMutator(entityTableName);
+ entityTable = new EntityTable().getTableMutator(hbaseConf, conn);
}
/**
@@ -86,9 +86,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
String flowName, String flowVersion, long flowRunId, String appId,
TimelineEntities data) throws IOException {
- byte[] rowKeyPrefix = TimelineWriterUtils.getRowKeyPrefix(clusterId,
- userId, flowName, flowRunId, appId);
-
TimelineWriteResponse putStatus = new TimelineWriteResponse();
for (TimelineEntity te : data.getEntities()) {
@@ -96,19 +93,19 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
if (te == null) {
continue;
}
- // get row key
- byte[] row = TimelineWriterUtils.join(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES, rowKeyPrefix,
- Bytes.toBytes(te.getType()), Bytes.toBytes(te.getId()));
-
- storeInfo(row, te, flowVersion);
- storeEvents(row, te.getEvents());
- storeConfig(row, te.getConfigs());
- storeMetrics(row, te.getMetrics());
- storeRelations(row, te.getIsRelatedToEntities(),
- EntityColumnDetails.PREFIX_IS_RELATED_TO);
- storeRelations(row, te.getRelatesToEntities(),
- EntityColumnDetails.PREFIX_RELATES_TO);
+
+ byte[] rowKey =
+ EntityRowKey.getRowKey(clusterId, userId, flowName, flowRunId, appId,
+ te);
+
+ storeInfo(rowKey, te, flowVersion);
+ storeEvents(rowKey, te.getEvents());
+ storeConfig(rowKey, te.getConfigs());
+ storeMetrics(rowKey, te.getMetrics());
+ storeRelations(rowKey, te.getIsRelatedToEntities(),
+ EntityColumnPrefix.IS_RELATED_TO);
+ storeRelations(rowKey, te.getRelatesToEntities(),
+ EntityColumnPrefix.RELATES_TO);
}
return putStatus;
@@ -119,10 +116,15 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
*/
private void storeRelations(byte[] rowKey,
Map<String, Set<String>> connectedEntities,
- EntityColumnDetails columnNamePrefix) throws IOException {
- for (Map.Entry<String, Set<String>> entry : connectedEntities.entrySet()) {
- columnNamePrefix.store(rowKey, entityTable, entry.getKey(),
- entry.getValue());
+ EntityColumnPrefix entityColumnPrefix) throws IOException {
+ for (Map.Entry<String, Set<String>> connectedEntity : connectedEntities
+ .entrySet()) {
+ // id3?id4?id5
+ String compoundValue =
+ Separator.VALUES.joinEncoded(connectedEntity.getValue());
+
+ entityColumnPrefix.store(rowKey, entityTable, connectedEntity.getKey(),
+ null, compoundValue);
}
}
@@ -132,13 +134,13 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion)
throws IOException {
- EntityColumnDetails.ID.store(rowKey, entityTable, te.getId());
- EntityColumnDetails.TYPE.store(rowKey, entityTable, te.getType());
- EntityColumnDetails.CREATED_TIME.store(rowKey, entityTable,
+ EntityColumn.ID.store(rowKey, entityTable, null, te.getId());
+ EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType());
+ EntityColumn.CREATED_TIME.store(rowKey, entityTable, null,
te.getCreatedTime());
- EntityColumnDetails.MODIFIED_TIME.store(rowKey, entityTable,
+ EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null,
te.getModifiedTime());
- EntityColumnDetails.FLOW_VERSION.store(rowKey, entityTable, flowVersion);
+ EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
}
/**
@@ -150,8 +152,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
return;
}
for (Map.Entry<String, String> entry : config.entrySet()) {
- EntityColumnFamily.CONFIG.store(rowKey, entityTable,
- entry.getKey(), entry.getValue());
+ EntityColumnPrefix.CONFIG.store(rowKey, entityTable, entry.getKey(),
+ null, entry.getValue());
}
}
@@ -163,11 +165,12 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
throws IOException {
if (metrics != null) {
for (TimelineMetric metric : metrics) {
- String key = metric.getId();
+ String metricColumnQualifier = metric.getId();
Map<Long, Number> timeseries = metric.getValues();
- for (Map.Entry<Long, Number> entry : timeseries.entrySet()) {
- EntityColumnFamily.METRICS.store(rowKey, entityTable, key,
- entry.getKey(), entry.getValue());
+ for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
+ Long timestamp = timeseriesEntry.getKey();
+ EntityColumnPrefix.METRIC.store(rowKey, entityTable,
+ metricColumnQualifier, timestamp, timeseriesEntry.getValue());
}
}
}
@@ -181,19 +184,27 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
if (events != null) {
for (TimelineEvent event : events) {
if (event != null) {
- String id = event.getId();
- if (id != null) {
- byte[] idBytes = Bytes.toBytes(id);
+ String eventId = event.getId();
+ if (eventId != null) {
Map<String, Object> eventInfo = event.getInfo();
if (eventInfo != null) {
for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
- EntityColumnDetails.PREFIX_EVENTS.store(rowKey,
- entityTable, idBytes, info.getKey(), info.getValue());
- }
+ // eventId?infoKey
+ byte[] columnQualifierFirst =
+ Bytes.toBytes(Separator.VALUES.encode(eventId));
+ byte[] compoundColumnQualifierBytes =
+ Separator.VALUES.join(columnQualifierFirst,
+ Bytes.toBytes(info.getKey()));
+ // convert back to string to avoid additional API on store.
+ String compoundColumnQualifier =
+ Bytes.toString(compoundColumnQualifierBytes);
+ EntityColumnPrefix.METRIC.store(rowKey, entityTable,
+ compoundColumnQualifier, null, info.getValue());
+ } // for info: eventInfo
}
}
}
- }
+ } // event : events
}
}
@@ -204,8 +215,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
}
/**
- * close the hbase connections
- * The close APIs perform flushing and release any
+ * close the hbase connections The close APIs perform flushing and release any
* resources held
*/
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
deleted file mode 100644
index 2a2db81..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class Range {
- private final int startIdx;
- private final int endIdx;
-
- /**
- * Defines a range from start index (inclusive) to end index (exclusive).
- *
- * @param start
- * Starting index position
- * @param end
- * Ending index position (exclusive)
- */
- public Range(int start, int end) {
- if (start < 0 || end < start) {
- throw new IllegalArgumentException(
- "Invalid range, required that: 0 <= start <= end; start=" + start
- + ", end=" + end);
- }
-
- this.startIdx = start;
- this.endIdx = end;
- }
-
- public int start() {
- return startIdx;
- }
-
- public int end() {
- return endIdx;
- }
-
- public int length() {
- return endIdx - startIdx;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
deleted file mode 100644
index d95cbb2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-
-/**
- * contains the constants used in the context of schema accesses for
- * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
- * information
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class TimelineEntitySchemaConstants {
-
- /** entity prefix */
- public static final String ENTITY_PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX
- + ".entity";
-
- /** config param name that specifies the entity table name */
- public static final String ENTITY_TABLE_NAME = ENTITY_PREFIX
- + ".table.name";
-
- /**
- * config param name that specifies the TTL for metrics column family in
- * entity table
- */
- public static final String ENTITY_TABLE_METRICS_TTL = ENTITY_PREFIX
- + ".table.metrics.ttl";
-
- /** default value for entity table name */
- public static final String DEFAULT_ENTITY_TABLE_NAME = "timelineservice.entity";
-
- /** in bytes default value for entity table name */
- static final byte[] DEFAULT_ENTITY_TABLE_NAME_BYTES = Bytes
- .toBytes(DEFAULT_ENTITY_TABLE_NAME);
-
- /** separator in row key */
- public static final String ROW_KEY_SEPARATOR = "!";
-
- /** byte representation of the separator in row key */
- static final byte[] ROW_KEY_SEPARATOR_BYTES = Bytes
- .toBytes(ROW_KEY_SEPARATOR);
-
- public static final byte ZERO_BYTES = 0;
-
- /** default TTL is 30 days for metrics timeseries */
- public static final int ENTITY_TABLE_METRICS_TTL_DEFAULT = 2592000;
-
- /** default max number of versions */
- public static final int ENTITY_TABLE_METRICS_MAX_VERSIONS_DEFAULT = 1000;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index 820a6d1..a5cc2ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -19,21 +19,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
@@ -41,7 +26,18 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
/**
* This creates the schema for a hbase based backend for storing application
@@ -53,18 +49,6 @@ public class TimelineSchemaCreator {
final static String NAME = TimelineSchemaCreator.class.getSimpleName();
private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
- final static byte[][] splits = { Bytes.toBytes("a"), Bytes.toBytes("ad"),
- Bytes.toBytes("an"), Bytes.toBytes("b"), Bytes.toBytes("ca"),
- Bytes.toBytes("cl"), Bytes.toBytes("d"), Bytes.toBytes("e"),
- Bytes.toBytes("f"), Bytes.toBytes("g"), Bytes.toBytes("h"),
- Bytes.toBytes("i"), Bytes.toBytes("j"), Bytes.toBytes("k"),
- Bytes.toBytes("l"), Bytes.toBytes("m"), Bytes.toBytes("n"),
- Bytes.toBytes("o"), Bytes.toBytes("q"), Bytes.toBytes("r"),
- Bytes.toBytes("s"), Bytes.toBytes("se"), Bytes.toBytes("t"),
- Bytes.toBytes("u"), Bytes.toBytes("v"), Bytes.toBytes("w"),
- Bytes.toBytes("x"), Bytes.toBytes("y"), Bytes.toBytes("z") };
-
- public static final String SPLIT_KEY_PREFIX_LENGTH = "4";
public static void main(String[] args) throws Exception {
@@ -79,13 +63,12 @@ public class TimelineSchemaCreator {
// Grab the entityTableName argument
String entityTableName = commandLine.getOptionValue("e");
if (StringUtils.isNotBlank(entityTableName)) {
- hbaseConf.set(TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
- entityTableName);
+ hbaseConf.set(EntityTable.TABLE_NAME_CONF_NAME, entityTableName);
}
- String entityTable_TTL_Metrics = commandLine.getOptionValue("m");
- if (StringUtils.isNotBlank(entityTable_TTL_Metrics)) {
- hbaseConf.set(TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL,
- entityTable_TTL_Metrics);
+ String entityTableTTLMetrics = commandLine.getOptionValue("m");
+ if (StringUtils.isNotBlank(entityTableTTLMetrics)) {
+ int metricsTTL = Integer.parseInt(entityTableTTLMetrics);
+ new EntityTable().setMetricsTTL(metricsTTL, hbaseConf);
}
createAllTables(hbaseConf);
}
@@ -136,7 +119,7 @@ public class TimelineSchemaCreator {
if (admin == null) {
throw new IOException("Cannot create table since admin is null");
}
- createTimelineEntityTable(admin, hbaseConf);
+ new EntityTable().createTable(admin, hbaseConf);
} finally {
if (conn != null) {
conn.close();
@@ -144,88 +127,5 @@ public class TimelineSchemaCreator {
}
}
- /**
- * Creates a table with column families info, config and metrics
- * info stores information about a timeline entity object
- * config stores configuration data of a timeline entity object
- * metrics stores the metrics of a timeline entity object
- *
- * Example entity table record:
- * <pre>
- *|------------------------------------------------------------|
- *| Row | Column Family | Column Family | Column Family|
- *| key | info | metrics | config |
- *|------------------------------------------------------------|
- *| userName! | id:entityId | metricName1: | configKey1: |
- *| clusterId! | | metricValue1 | configValue1 |
- *| flowId! | type:entityType| @timestamp1 | |
- *| flowRunId! | | | configKey2: |
- *| AppId! | created_time: | metricName1: | configValue2 |
- *| entityType!| 1392993084018 | metricValue2 | |
- *| entityId | | @timestamp2 | |
- *| | modified_time: | | |
- *| | 1392995081012 | metricName2: | |
- *| | | metricValue1 | |
- *| | r!relatesToKey:| @timestamp2 | |
- *| | id3!id4!id5 | | |
- *| | | | |
- *| | s!isRelatedToKey| | |
- *| | id7!id9!id5 | | |
- *| | | | |
- *| | e!eventKey: | | |
- *| | eventValue | | |
- *| | | | |
- *| | flowVersion: | | |
- *| | versionValue | | |
- *|------------------------------------------------------------|
- *</pre>
- * @param admin
- * @param hbaseConf
- * @throws IOException
- */
- public static void createTimelineEntityTable(Admin admin,
- Configuration hbaseConf) throws IOException {
-
- TableName table = TableName.valueOf(hbaseConf.get(
- TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
- TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME));
- if (admin.tableExists(table)) {
- // do not disable / delete existing table
- // similar to the approach taken by map-reduce jobs when
- // output directory exists
- throw new IOException("Table " + table.getNameAsString()
- + " already exists.");
- }
-
- HTableDescriptor entityTableDescp = new HTableDescriptor(table);
- HColumnDescriptor cf1 = new HColumnDescriptor(
- EntityColumnFamily.INFO.getInBytes());
- cf1.setBloomFilterType(BloomType.ROWCOL);
- entityTableDescp.addFamily(cf1);
-
- HColumnDescriptor cf2 = new HColumnDescriptor(
- EntityColumnFamily.CONFIG.getInBytes());
- cf2.setBloomFilterType(BloomType.ROWCOL);
- cf2.setBlockCacheEnabled(true);
- entityTableDescp.addFamily(cf2);
-
- HColumnDescriptor cf3 = new HColumnDescriptor(
- EntityColumnFamily.METRICS.getInBytes());
- entityTableDescp.addFamily(cf3);
- cf3.setBlockCacheEnabled(true);
- // always keep 1 version (the latest)
- cf3.setMinVersions(1);
- cf3.setMaxVersions(TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_MAX_VERSIONS_DEFAULT);
- cf3.setTimeToLive(hbaseConf.getInt(
- TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL,
- TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL_DEFAULT));
- entityTableDescp
- .setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
- entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
- SPLIT_KEY_PREFIX_LENGTH);
- admin.createTable(entityTableDescp, splits);
- LOG.info("Status of table creation for " + table.getNameAsString() + "="
- + admin.tableExists(table));
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
index 467bcec..494e8ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
@@ -15,17 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.service.Service;
/**
* This interface is for storing application timeline information.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
deleted file mode 100644
index 113935e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.Range;
-
-/**
- * bunch of utility functions used across TimelineWriter classes
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class TimelineWriterUtils {
-
- /** empty bytes */
- public static final byte[] EMPTY_BYTES = new byte[0];
- private static final String SPACE = " ";
- private static final String UNDERSCORE = "_";
- private static final String EMPTY_STRING = "";
-
- /**
- * Returns a single byte array containing all of the individual component
- * arrays separated by the separator array.
- *
- * @param separator
- * @param components
- * @return byte array after joining the components
- */
- public static byte[] join(byte[] separator, byte[]... components) {
- if (components == null || components.length == 0) {
- return EMPTY_BYTES;
- }
-
- int finalSize = 0;
- if (separator != null) {
- finalSize = separator.length * (components.length - 1);
- }
- for (byte[] comp : components) {
- if (comp != null) {
- finalSize += comp.length;
- }
- }
-
- byte[] buf = new byte[finalSize];
- int offset = 0;
- for (int i = 0; i < components.length; i++) {
- if (components[i] != null) {
- System.arraycopy(components[i], 0, buf, offset, components[i].length);
- offset += components[i].length;
- if (i < (components.length - 1) && separator != null
- && separator.length > 0) {
- System.arraycopy(separator, 0, buf, offset, separator.length);
- offset += separator.length;
- }
- }
- }
- return buf;
- }
-
- /**
- * Splits the source array into multiple array segments using the given
- * separator, up to a maximum of count items. This will naturally produce
- * copied byte arrays for each of the split segments. To identify the split
- * ranges without the array copies, see
- * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
- *
- * @param source
- * @param separator
- * @return byte[] array after splitting the source
- */
- public static byte[][] split(byte[] source, byte[] separator) {
- return split(source, separator, -1);
- }
-
- /**
- * Splits the source array into multiple array segments using the given
- * separator, up to a maximum of count items. This will naturally produce
- * copied byte arrays for each of the split segments. To identify the split
- * ranges without the array copies, see
- * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
- *
- * @param source
- * @param separator
- * @param limit
- * @return byte[][] after splitting the input source
- */
- public static byte[][] split(byte[] source, byte[] separator, int limit) {
- List<Range> segments = splitRanges(source, separator, limit);
-
- byte[][] splits = new byte[segments.size()][];
- for (int i = 0; i < segments.size(); i++) {
- Range r = segments.get(i);
- byte[] tmp = new byte[r.length()];
- if (tmp.length > 0) {
- System.arraycopy(source, r.start(), tmp, 0, r.length());
- }
- splits[i] = tmp;
- }
- return splits;
- }
-
- /**
- * Returns a list of ranges identifying [start, end) -- closed, open --
- * positions within the source byte array that would be split using the
- * separator byte array.
- */
- public static List<Range> splitRanges(byte[] source, byte[] separator) {
- return splitRanges(source, separator, -1);
- }
-
- /**
- * Returns a list of ranges identifying [start, end) -- closed, open --
- * positions within the source byte array that would be split using the
- * separator byte array.
- * @param source the source data
- * @param separator the separator pattern to look for
- * @param limit the maximum number of splits to identify in the source
- */
- public static List<Range> splitRanges(byte[] source, byte[] separator, int limit) {
- List<Range> segments = new ArrayList<Range>();
- if ((source == null) || (separator == null)) {
- return segments;
- }
- int start = 0;
- itersource: for (int i = 0; i < source.length; i++) {
- for (int j = 0; j < separator.length; j++) {
- if (source[i + j] != separator[j]) {
- continue itersource;
- }
- }
- // all separator elements matched
- if (limit > 0 && segments.size() >= (limit-1)) {
- // everything else goes in one final segment
- break;
- }
-
- segments.add(new Range(start, i));
- start = i + separator.length;
- // i will be incremented again in outer for loop
- i += separator.length-1;
- }
- // add in remaining to a final range
- if (start <= source.length) {
- segments.add(new Range(start, source.length));
- }
- return segments;
- }
-
- /**
- * converts run id into it's inverse timestamp
- * @param flowRunId
- * @return inverted long
- */
- public static long encodeRunId(Long flowRunId) {
- return Long.MAX_VALUE - flowRunId;
- }
-
- /**
- * return a value from the Map as a String
- * @param key
- * @param values
- * @return value as a String or ""
- * @throws IOException
- */
- public static String getValueAsString(final byte[] key,
- final Map<byte[], byte[]> values) throws IOException {
- if( values == null ) {
- return EMPTY_STRING;
- }
- byte[] value = values.get(key);
- if (value != null) {
- return GenericObjectMapper.read(value).toString();
- } else {
- return EMPTY_STRING;
- }
- }
-
- /**
- * return a value from the Map as a long
- * @param key
- * @param values
- * @return value as Long or 0L
- * @throws IOException
- */
- public static long getValueAsLong(final byte[] key,
- final Map<byte[], byte[]> values) throws IOException {
- if (values == null) {
- return 0;
- }
- byte[] value = values.get(key);
- if (value != null) {
- Number val = (Number) GenericObjectMapper.read(value);
- return val.longValue();
- } else {
- return 0L;
- }
- }
-
- /**
- * concates the values from a Set<Strings> to return a single delimited string value
- * @param rowKeySeparator
- * @param values
- * @return Value from the set of strings as a string
- */
- public static String getValueAsString(String rowKeySeparator,
- Set<String> values) {
-
- if (values == null) {
- return EMPTY_STRING;
- }
- StringBuilder concatStrings = new StringBuilder();
- for (String value : values) {
- concatStrings.append(value);
- concatStrings.append(rowKeySeparator);
- }
- // remove the last separator
- if(concatStrings.length() > 1) {
- concatStrings.deleteCharAt(concatStrings.lastIndexOf(rowKeySeparator));
- }
- return concatStrings.toString();
- }
- /**
- * Constructs a row key prefix for the entity table
- * @param clusterId
- * @param userId
- * @param flowId
- * @param flowRunId
- * @param appId
- * @return byte array with the row key prefix
- */
- static byte[] getRowKeyPrefix(String clusterId, String userId, String flowId,
- Long flowRunId, String appId) {
- return TimelineWriterUtils.join(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
- Bytes.toBytes(cleanse(userId)), Bytes.toBytes(cleanse(clusterId)),
- Bytes.toBytes(cleanse(flowId)),
- Bytes.toBytes(TimelineWriterUtils.encodeRunId(flowRunId)),
- Bytes.toBytes(cleanse(appId)));
- }
-
- /**
- * Takes a string token to be used as a key or qualifier and
- * cleanses out reserved tokens.
- * This operation is not symmetrical.
- * Logic is to replace all spaces and separator chars in input with
- * underscores.
- *
- * @param token token to cleanse.
- * @return String with no spaces and no separator chars
- */
- public static String cleanse(String token) {
- if (token == null || token.length() == 0) {
- return token;
- }
-
- String cleansed = token.replaceAll(SPACE, UNDERSCORE);
- cleansed = cleansed.replaceAll(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR, UNDERSCORE);
-
- return cleansed;
- }
-
- /**
- * stores the info to the table in hbase
- *
- * @param rowKey
- * @param table
- * @param columnFamily
- * @param columnPrefix
- * @param columnQualifier
- * @param inputValue
- * @param cellTimeStamp
- * @throws IOException
- */
- public static void store(byte[] rowKey, BufferedMutator table, byte[] columnFamily,
- byte[] columnPrefix, byte[] columnQualifier, Object inputValue,
- Long cellTimeStamp) throws IOException {
- if ((rowKey == null) || (table == null) || (columnFamily == null)
- || (columnQualifier == null) || (inputValue == null)) {
- return;
- }
-
- Put p = null;
- if (cellTimeStamp == null) {
- if (columnPrefix != null) {
- // store with prefix
- p = new Put(rowKey);
- p.addColumn(
- columnFamily,
- join(TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
- columnPrefix, columnQualifier), GenericObjectMapper
- .write(inputValue));
- } else {
- // store without prefix
- p = new Put(rowKey);
- p.addColumn(columnFamily, columnQualifier,
- GenericObjectMapper.write(inputValue));
- }
- } else {
- // store with cell timestamp
- Cell cell = CellUtil.createCell(rowKey, columnFamily, columnQualifier,
- // set the cell timestamp
- cellTimeStamp,
- // KeyValue Type minimum
- TimelineEntitySchemaConstants.ZERO_BYTES,
- GenericObjectMapper.write(inputValue));
- p = new Put(rowKey);
- p.add(cell);
- }
- if (p != null) {
- table.mutate(p);
- }
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
new file mode 100644
index 0000000..e8d8b5c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+
+/**
+ * Implements behavior common to tables used in the timeline service storage.
+ *
+ * @param <T> reference to the table instance class itself for type safety.
+ */
+public abstract class BaseTable<T> {
+
+ /**
+ * Name of config variable that is used to point to this table
+ */
+ private final String tableNameConfName;
+
+ /**
+ * Unless the configuration overrides, this will be the default name for the
+ * table when it is created.
+ */
+ private final String defaultTableName;
+
+ /**
+ * @param tableNameConfName name of config variable that is used to point to
+ * this table.
+ */
+ protected BaseTable(String tableNameConfName, String defaultTableName) {
+ this.tableNameConfName = tableNameConfName;
+ this.defaultTableName = defaultTableName;
+ }
+
+ /**
+ * Used to create a type-safe mutator for this table.
+ *
+ * @param hbaseConf used to read table name
+ * @param conn used to create a table from.
+ * @return a type safe {@link BufferedMutator} for the entity table.
+ * @throws IOException
+ */
+ public TypedBufferedMutator<T> getTableMutator(Configuration hbaseConf,
+ Connection conn) throws IOException {
+
+ TableName tableName = this.getTableName(hbaseConf);
+
+ // Plain buffered mutator
+ BufferedMutator bufferedMutator = conn.getBufferedMutator(tableName);
+
+ // Now make this thing type safe.
+ // This is how service initialization should hang on to this variable, with
+ // the proper type
+ TypedBufferedMutator<T> table =
+ new BufferedMutatorDelegator<T>(bufferedMutator);
+
+ return table;
+ }
+
+ /**
+ * @param hbaseConf used to read settings that override defaults
+ * @param conn used to create table from
+ * @param scan that specifies what you want to read from this table.
+ * @return scanner for the table.
+ * @throws IOException
+ */
+ public ResultScanner getResultScanner(Configuration hbaseConf,
+ Connection conn, Scan scan) throws IOException {
+ Table table = conn.getTable(getTableName(hbaseConf));
+ return table.getScanner(scan);
+ }
+
+ /**
+ * Get the table name for this table.
+ *
+ * @param hbaseConf
+ */
+ public TableName getTableName(Configuration hbaseConf) {
+ TableName table =
+ TableName.valueOf(hbaseConf.get(tableNameConfName, defaultTableName));
+ return table;
+
+ }
+
+ /**
+ * Used to create the table in HBase. Should be called only once (per HBase
+ * instance).
+ *
+ * @param admin
+ * @param hbaseConf
+ */
+ public abstract void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
new file mode 100644
index 0000000..fe8f9c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Mutation;
+
+/**
+ * To be used to wrap an actual {@link BufferedMutator} in a type safe manner
+ *
+ * @param <T> The class referring to the table to be written to.
+ */
+class BufferedMutatorDelegator<T> implements TypedBufferedMutator<T> {
+
+ private final BufferedMutator bufferedMutator;
+
+ /**
+ * @param bufferedMutator the mutator to be wrapped for delegation. Shall not
+ * be null.
+ */
+ public BufferedMutatorDelegator(BufferedMutator bufferedMutator) {
+ this.bufferedMutator = bufferedMutator;
+ }
+
+ public TableName getName() {
+ return bufferedMutator.getName();
+ }
+
+ public Configuration getConfiguration() {
+ return bufferedMutator.getConfiguration();
+ }
+
+ public void mutate(Mutation mutation) throws IOException {
+ bufferedMutator.mutate(mutation);
+ }
+
+ public void mutate(List<? extends Mutation> mutations) throws IOException {
+ bufferedMutator.mutate(mutations);
+ }
+
+ public void close() throws IOException {
+ bufferedMutator.close();
+ }
+
+ public void flush() throws IOException {
+ bufferedMutator.flush();
+ }
+
+ public long getWriteBufferSize() {
+ return bufferedMutator.getWriteBufferSize();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
new file mode 100644
index 0000000..3397d62
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Result;
+
+/**
+ * A Column represents the way to store a fully qualified column in a specific
+ * table.
+ */
+public interface Column<T> {
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey identifying the row to write. Nothing gets written when null.
+ * @param tableMutator used to modify the underlying HBase table. Caller is
+ * responsible to pass a mutator for the table that actually has this
+ * column.
+ * @param timestamp version timestamp. When null the server timestamp will be
+ * used.
+ * @param inputValue the value to write to the rowKey and column qualifier.
+ * Nothing gets written when null.
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
+ Long timestamp, Object inputValue) throws IOException;
+
+ /**
+ * Get the latest version of this specified column. Note: this call clones the
+ * value content of the hosting {@link Cell}.
+ *
+ * @param result Cannot be null
+ * @return result object (can be cast to whatever object was written to), or
+ * null when result doesn't contain this column.
+ * @throws IOException
+ */
+ public Object readResult(Result result) throws IOException;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
new file mode 100644
index 0000000..c84c016
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+/**
+ * Type safe column family.
+ *
+ * @param <T> refers to the table for which this column family is used for.
+ */
+public interface ColumnFamily<T> {
+
+ /**
+ * Keep a local copy if you need to avoid overhead of repeated cloning.
+ *
+ * @return a clone of the byte representation of the column family.
+ */
+ public byte[] getBytes();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
new file mode 100644
index 0000000..6a204dc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+
+/**
+ * This class is meant to be used only by explicit Columns, and not directly to
+ * write by clients.
+ *
+ * @param <T> refers to the table.
+ */
+public class ColumnHelper<T> {
+
+ private final ColumnFamily<T> columnFamily;
+
+ /**
+ * Local copy of bytes representation of columnFamily so that we can avoid
+ * cloning a new copy over and over.
+ */
+ private final byte[] columnFamilyBytes;
+
+ public ColumnHelper(ColumnFamily<T> columnFamily) {
+ this.columnFamily = columnFamily;
+ columnFamilyBytes = columnFamily.getBytes();
+ }
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey identifying the row to write. Nothing gets written when null.
+ * @param tableMutator used to modify the underlying HBase table
+ * @param columnQualifier column qualifier. Nothing gets written when null.
+ * @param timestamp version timestamp. When null the server timestamp will be
+ * used.
+ * @param inputValue the value to write to the rowKey and column qualifier.
+ * Nothing gets written when null.
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, TypedBufferedMutator<?> tableMutator,
+ byte[] columnQualifier, Long timestamp, Object inputValue)
+ throws IOException {
+ if ((rowKey == null) || (columnQualifier == null) || (inputValue == null)) {
+ return;
+ }
+ Put p = new Put(rowKey);
+
+ if (timestamp == null) {
+ p.addColumn(columnFamilyBytes, columnQualifier,
+ GenericObjectMapper.write(inputValue));
+ } else {
+ p.addColumn(columnFamilyBytes, columnQualifier, timestamp,
+ GenericObjectMapper.write(inputValue));
+ }
+ tableMutator.mutate(p);
+ }
+
+ /**
+ * @return the column family for this column implementation.
+ */
+ public ColumnFamily<T> getColumnFamily() {
+ return columnFamily;
+ }
+
+ /**
+ * Get the latest version of this specified column. Note: this call clones the
+ * value content of the hosting {@link Cell}.
+ *
+ * @param result from which to read the value. Cannot be null
+ * @param columnQualifierBytes referring to the column to be read.
+ * @return latest version of the specified column of whichever object was
+ * written.
+ * @throws IOException
+ */
+ public Object readResult(Result result, byte[] columnQualifierBytes)
+ throws IOException {
+ if (result == null || columnQualifierBytes == null) {
+ return null;
+ }
+
+ // Would have preferred to be able to use getValueAsByteBuffer and get a
+ // ByteBuffer to avoid copy, but GenericObjectMapper doesn't seem to like
+ // that.
+ byte[] value = result.getValue(columnFamilyBytes, columnQualifierBytes);
+ return GenericObjectMapper.read(value);
+ }
+
+ /**
+ * @param result from which to reads timeseries data
+ * @param columnPrefixBytes optional prefix to limit columns. If null all
+ * columns are returned.
+ * @return the cell values at each respective time in for form
+ * {idA={timestamp1->value1}, idA={timestamp2->value2},
+ * idB={timestamp3->value3}, idC={timestamp1->value4}}
+ * @throws IOException
+ */
+ public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
+ Result result, byte[] columnPrefixBytes) throws IOException {
+
+ NavigableMap<String, NavigableMap<Long, Number>> results =
+ new TreeMap<String, NavigableMap<Long, Number>>();
+
+ if (result != null) {
+ NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
+ result.getMap();
+
+ NavigableMap<byte[], NavigableMap<Long, byte[]>> columnCellMap =
+ resultMap.get(columnFamilyBytes);
+
+ // could be that there is no such column family.
+ if (columnCellMap != null) {
+ for (Entry<byte[], NavigableMap<Long, byte[]>> entry : columnCellMap
+ .entrySet()) {
+ String columnName = null;
+ if (columnPrefixBytes == null) {
+ // Decode the spaces we encoded in the column name.
+ columnName = Separator.decode(entry.getKey(), Separator.SPACE);
+ } else {
+ // A non-null prefix means columns are actually of the form
+ // prefix!columnNameRemainder
+ byte[][] columnNameParts =
+ Separator.QUALIFIERS.split(entry.getKey(), 2);
+ byte[] actualColumnPrefixBytes = columnNameParts[0];
+ if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
+ && columnNameParts.length == 2) {
+ // This is the prefix that we want
+ columnName = Separator.decode(columnNameParts[1]);
+ }
+ }
+
+ // If this column has the prefix we want
+ if (columnName != null) {
+ NavigableMap<Long, Number> cellResults =
+ new TreeMap<Long, Number>();
+ NavigableMap<Long, byte[]> cells = entry.getValue();
+ if (cells != null) {
+ for (Entry<Long, byte[]> cell : cells.entrySet()) {
+ Number value =
+ (Number) GenericObjectMapper.read(cell.getValue());
+ cellResults.put(cell.getKey(), value);
+ }
+ }
+ results.put(columnName, cellResults);
+ }
+ } // for entry : columnCellMap
+ } // if columnCellMap != null
+ } // if result != null
+ return results;
+ }
+
+ /**
+ * @param result from which to read columns
+ * @param columnPrefixBytes optional prefix to limit columns. If null all
+ * columns are returned.
+ * @return the latest values of columns in the column family.
+ * @throws IOException
+ */
+ public Map<String, Object> readResults(Result result, byte[] columnPrefixBytes)
+ throws IOException {
+ Map<String, Object> results = new HashMap<String, Object>();
+
+ if (result != null) {
+ Map<byte[], byte[]> columns = result.getFamilyMap(columnFamilyBytes);
+ for (Entry<byte[], byte[]> entry : columns.entrySet()) {
+ if (entry.getKey() != null && entry.getKey().length > 0) {
+
+ String columnName = null;
+ if (columnPrefixBytes == null) {
+ // Decode the spaces we encoded in the column name.
+ columnName = Separator.decode(entry.getKey(), Separator.SPACE);
+ } else {
+ // A non-null prefix means columns are actually of the form
+ // prefix!columnNameRemainder
+ byte[][] columnNameParts =
+ Separator.QUALIFIERS.split(entry.getKey(), 2);
+ byte[] actualColumnPrefixBytes = columnNameParts[0];
+ if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
+ && columnNameParts.length == 2) {
+ // This is the prefix that we want
+ columnName = Separator.decode(columnNameParts[1]);
+ }
+ }
+
+ // If this column has the prefix we want
+ if (columnName != null) {
+ Object value = GenericObjectMapper.read(entry.getValue());
+ results.put(columnName, value);
+ }
+ }
+ } // for entry
+ }
+ return results;
+ }
+
+ /**
+ * @param columnPrefixBytes The byte representation for the column prefix.
+ * Should not contain {@link Separator#QUALIFIERS}.
+ * @param qualifier for the remainder of the column. Any
+ * {@link Separator#QUALIFIERS} will be encoded in the qualifier.
+ * @return fully sanitized column qualifier that is a combination of prefix
+ * and qualifier. If prefix is null, the result is simply the encoded
+ * qualifier without any separator.
+ */
+ public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
+ String qualifier) {
+
+ // We don't want column names to have spaces
+ byte[] encodedQualifier = Bytes.toBytes(Separator.SPACE.encode(qualifier));
+ if (columnPrefixBytes == null) {
+ return encodedQualifier;
+ }
+
+ // Convert qualifier to lower case, strip of separators and tag on column
+ // prefix.
+ byte[] columnQualifier =
+ Separator.QUALIFIERS.join(columnPrefixBytes, encodedQualifier);
+ return columnQualifier;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
new file mode 100644
index 0000000..2eedea0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Result;
+
+/**
+ * Used to represent a partially qualified column, where the actual column name
+ * will be composed of a prefix and the remainder of the column qualifier. The
+ * prefix can be null, in which case the column qualifier will be completely
+ * determined when the values are stored.
+ */
+public interface ColumnPrefix<T> {
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey identifying the row to write. Nothing gets written when null.
+ * @param tableMutator used to modify the underlying HBase table. Caller is
+ * responsible to pass a mutator for the table that actually has this
+ * column.
+ * @param qualifier column qualifier. Nothing gets written when null.
+ * @param timestamp version timestamp. When null the server timestamp will be
+ * used.
+ * @param inputValue the value to write to the rowKey and column qualifier.
+ * Nothing gets written when null.
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
+ String qualifier, Long timestamp, Object inputValue) throws IOException;
+
+ /**
+ * Get the latest version of this specified column. Note: this call clones the
+ * value content of the hosting {@link Cell}.
+ *
+ * @param result Cannot be null
+ * @param qualifier column qualifier. Nothing gets read when null.
+ * @return result object (can be cast to whatever object was written to) or
+ * null when specified column qualifier for this prefix doesn't exist
+ * in the result.
+ * @throws IOException
+ */
+ public Object readResult(Result result, String qualifier) throws IOException;
+
+ /**
+ * @param resultfrom which to read columns
+ * @return the latest values of columns in the column family with this prefix
+ * (or all of them if the prefix value is null).
+ * @throws IOException
+ */
+ public Map<String, Object> readResults(Result result) throws IOException;
+
+ /**
+ * @param result from which to reads timeseries data
+ * @return the cell values at each respective time in for form
+ * {idA={timestamp1->value1}, idA={timestamp2->value2},
+ * idB={timestamp3->value3}, idC={timestamp1->value4}}
+ * @throws IOException
+ */
+ public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
+ Result result) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
new file mode 100644
index 0000000..2cb6c08
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Range {
+ private final int startIdx;
+ private final int endIdx;
+
+ /**
+ * Defines a range from start index (inclusive) to end index (exclusive).
+ *
+ * @param start
+ * Starting index position
+ * @param end
+ * Ending index position (exclusive)
+ */
+ public Range(int start, int end) {
+ if (start < 0 || end < start) {
+ throw new IllegalArgumentException(
+ "Invalid range, required that: 0 <= start <= end; start=" + start
+ + ", end=" + end);
+ }
+
+ this.startIdx = start;
+ this.endIdx = end;
+ }
+
+ public int start() {
+ return startIdx;
+ }
+
+ public int end() {
+ return endIdx;
+ }
+
+ public int length() {
+ return endIdx - startIdx;
+ }
+}
\ No newline at end of file
[36/50] [abbrv] hadoop git commit: YARN-3814. REST API implementation
for getting raw entities in TimelineReader (Varun Saxena via sjlee)
Posted by sj...@apache.org.
YARN-3814. REST API implementation for getting raw entities in TimelineReader (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a34844e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a34844e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a34844e
Branch: refs/heads/YARN-2928-rebase
Commit: 5a34844e82ab2153c2b79483de64ade5d880364f
Parents: a81fa64
Author: Sangjin Lee <sj...@apache.org>
Authored: Fri Aug 21 19:10:23 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 16:50:34 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../reader/TimelineReaderManager.java | 41 ++
.../reader/TimelineReaderServer.java | 2 +-
.../reader/TimelineReaderWebServices.java | 245 +++++++++-
.../storage/FileSystemTimelineReaderImpl.java | 5 +
.../reader/TestTimelineReaderWebServices.java | 456 ++++++++++++++++++-
6 files changed, 741 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a34844e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index aa5dde2..75af072 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -97,6 +97,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-4025. Deal with byte representations of Longs in writer code.
(Sangjin Lee and Vrushali C via junping_du)
+ YARN-3814. REST API implementation for getting raw entities in
+ TimelineReader (Varun Saxena via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a34844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
index 5573185..7fafd82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -18,10 +18,18 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
@Private
@Unstable
@@ -33,4 +41,37 @@ public class TimelineReaderManager extends AbstractService {
super(TimelineReaderManager.class.getName());
this.reader = timelineReader;
}
+
+ /**
+ * Get a set of entities matching given predicates. The meaning of each
+ * argument has been documented with {@link TimelineReader#getEntities}.
+ *
+ * @see TimelineReader#getEntities
+ */
+ Set<TimelineEntity> getEntities(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException {
+ return reader.getEntities(userId, clusterId, flowId, flowRunId, appId,
+ entityType, limit, createdTimeBegin, createdTimeEnd, modifiedTimeBegin,
+ modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters,
+ metricFilters, eventFilters, fieldsToRetrieve);
+ }
+
+ /**
+ * Get single timeline entity. The meaning of each argument has been
+ * documented with {@link TimelineReader#getEntity}.
+ *
+ * @see TimelineReader#getEntity
+ */
+ public TimelineEntity getEntity(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fields) throws IOException {
+ return reader.getEntity(userId, clusterId, flowId, flowRunId, appId,
+ entityType, entityId, fields);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a34844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
index 874112c..319cfb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -54,7 +54,7 @@ import com.google.common.annotations.VisibleForTesting;
public class TimelineReaderServer extends CompositeService {
private static final Log LOG = LogFactory.getLog(TimelineReaderServer.class);
private static final int SHUTDOWN_HOOK_PRIORITY = 30;
- private static final String TIMELINE_READER_MANAGER_ATTR =
+ static final String TIMELINE_READER_MANAGER_ATTR =
"timeline.reader.manager";
private HttpServer2 readerWebServer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a34844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index 3655a72..0b5fde0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -18,42 +18,283 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
import com.google.inject.Singleton;
-
/** REST end point for Timeline Reader */
@Private
@Unstable
@Singleton
@Path("/ws/v2/timeline")
public class TimelineReaderWebServices {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineReaderWebServices.class);
+
+ @Context private ServletContext ctxt;
+
+ private static final String COMMA_DELIMITER = ",";
+ private static final String COLON_DELIMITER = ":";
private void init(HttpServletResponse response) {
response.setContentType(null);
}
+ private static Set<String> parseValuesStr(String str, String delimiter) {
+ if (str == null || str.isEmpty()) {
+ return null;
+ }
+ Set<String> strSet = new HashSet<String>();
+ String[] strs = str.split(delimiter);
+ for (String aStr : strs) {
+ strSet.add(aStr.trim());
+ }
+ return strSet;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <T> void parseKeyValues(Map<String,T> map, String str,
+ String pairsDelim, String keyValuesDelim, boolean stringValue,
+ boolean multipleValues) {
+ String[] pairs = str.split(pairsDelim);
+ for (String pair : pairs) {
+ if (pair == null || pair.trim().isEmpty()) {
+ continue;
+ }
+ String[] pairStrs = pair.split(keyValuesDelim);
+ if (pairStrs.length < 2) {
+ continue;
+ }
+ if (!stringValue) {
+ try {
+ Object value =
+ GenericObjectMapper.OBJECT_READER.readValue(pairStrs[1].trim());
+ map.put(pairStrs[0].trim(), (T) value);
+ } catch (IOException e) {
+ map.put(pairStrs[0].trim(), (T) pairStrs[1].trim());
+ }
+ } else {
+ String key = pairStrs[0].trim();
+ if (multipleValues) {
+ Set<String> values = new HashSet<String>();
+ for (int i = 1; i < pairStrs.length; i++) {
+ values.add(pairStrs[i].trim());
+ }
+ map.put(key, (T) values);
+ } else {
+ map.put(key, (T) pairStrs[1].trim());
+ }
+ }
+ }
+ }
+
+ private static Map<String, Set<String>> parseKeyStrValuesStr(String str,
+ String pairsDelim, String keyValuesDelim) {
+ if (str == null) {
+ return null;
+ }
+ Map<String, Set<String>> map = new HashMap<String, Set<String>>();
+ parseKeyValues(map, str,pairsDelim, keyValuesDelim, true, true);
+ return map;
+ }
+
+ private static Map<String, String> parseKeyStrValueStr(String str,
+ String pairsDelim, String keyValDelim) {
+ if (str == null) {
+ return null;
+ }
+ Map<String, String> map = new HashMap<String, String>();
+ parseKeyValues(map, str, pairsDelim, keyValDelim, true, false);
+ return map;
+ }
+
+ private static Map<String, Object> parseKeyStrValueObj(String str,
+ String pairsDelim, String keyValDelim) {
+ if (str == null) {
+ return null;
+ }
+ Map<String, Object> map = new HashMap<String, Object>();
+ parseKeyValues(map, str, pairsDelim, keyValDelim, false, false);
+ return map;
+ }
+
+ private static EnumSet<Field> parseFieldsStr(String str, String delimiter) {
+ if (str == null) {
+ return null;
+ }
+ String[] strs = str.split(delimiter);
+ EnumSet<Field> fieldList = EnumSet.noneOf(Field.class);
+ for (String s : strs) {
+ fieldList.add(Field.valueOf(s.trim().toUpperCase()));
+ }
+ return fieldList;
+ }
+
+ private static Long parseLongStr(String str) {
+ return str == null ? null : Long.parseLong(str.trim());
+ }
+
+ private static String parseStr(String str) {
+ return str == null ? null : str.trim();
+ }
+
+ private static UserGroupInformation getUser(HttpServletRequest req) {
+ String remoteUser = req.getRemoteUser();
+ UserGroupInformation callerUGI = null;
+ if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+ return callerUGI;
+ }
+
+ private TimelineReaderManager getTimelineReaderManager() {
+ return (TimelineReaderManager)
+ ctxt.getAttribute(TimelineReaderServer.TIMELINE_READER_MANAGER_ATTR);
+ }
+
/**
* Return the description of the timeline reader web services.
*/
@GET
- @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ @Produces(MediaType.APPLICATION_JSON)
public TimelineAbout about(
@Context HttpServletRequest req,
@Context HttpServletResponse res) {
init(res);
return TimelineUtils.createTimelineAbout("Timeline Reader API");
}
+
+ /**
+ * Return a set of entities that match the given parameters.
+ */
+ @GET
+ @Path("/entities/{clusterId}/{appId}/{entityType}")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getEntities(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("clusterId") String clusterId,
+ @PathParam("appId") String appId,
+ @PathParam("entityType") String entityType,
+ @QueryParam("userId") String userId,
+ @QueryParam("flowId") String flowId,
+ @QueryParam("flowRunId") String flowRunId,
+ @QueryParam("limit") String limit,
+ @QueryParam("createdTimeStart") String createdTimeStart,
+ @QueryParam("createdTimeEnd") String createdTimeEnd,
+ @QueryParam("modifiedTimeStart") String modifiedTimeStart,
+ @QueryParam("modifiedTimeEnd") String modifiedTimeEnd,
+ @QueryParam("relatesto") String relatesTo,
+ @QueryParam("isrelatedto") String isRelatedTo,
+ @QueryParam("infofilters") String infofilters,
+ @QueryParam("conffilters") String conffilters,
+ @QueryParam("metricfilters") String metricfilters,
+ @QueryParam("eventfilters") String eventfilters,
+ @QueryParam("fields") String fields) {
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ UserGroupInformation callerUGI = getUser(req);
+ try {
+ return timelineReaderManager.getEntities(
+ callerUGI != null && (userId == null || userId.isEmpty()) ?
+ callerUGI.getUserName().trim() : parseStr(userId),
+ parseStr(clusterId), parseStr(flowId),
+ parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
+ parseLongStr(limit), parseLongStr(createdTimeStart),
+ parseLongStr(createdTimeEnd), parseLongStr(modifiedTimeStart),
+ parseLongStr(modifiedTimeEnd),
+ parseKeyStrValuesStr(relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
+ parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
+ parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER),
+ parseKeyStrValueStr(conffilters, COMMA_DELIMITER, COLON_DELIMITER),
+ parseValuesStr(metricfilters, COMMA_DELIMITER),
+ parseValuesStr(eventfilters, COMMA_DELIMITER),
+ parseFieldsStr(fields, COMMA_DELIMITER));
+ } catch (NumberFormatException e) {
+ throw new BadRequestException(
+ "createdTime or modifiedTime start/end or limit or flowId is not" +
+ " a numeric value.");
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("Requested Invalid Field.");
+ } catch (Exception e) {
+ LOG.error("Error getting entities", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ /**
+ * Return a single entity of the given entity type and Id.
+ */
+ @GET
+ @Path("/entity/{clusterId}/{appId}/{entityType}/{entityId}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public TimelineEntity getEntity(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("clusterId") String clusterId,
+ @PathParam("appId") String appId,
+ @PathParam("entityType") String entityType,
+ @PathParam("entityId") String entityId,
+ @QueryParam("userId") String userId,
+ @QueryParam("flowId") String flowId,
+ @QueryParam("flowRunId") String flowRunId,
+ @QueryParam("fields") String fields) {
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ UserGroupInformation callerUGI = getUser(req);
+ TimelineEntity entity = null;
+ try {
+ entity = timelineReaderManager.getEntity(
+ callerUGI != null && (userId == null || userId.isEmpty()) ?
+ callerUGI.getUserName().trim() : parseStr(userId),
+ parseStr(clusterId), parseStr(flowId), parseLongStr(flowRunId),
+ parseStr(appId), parseStr(entityType), parseStr(entityId),
+ parseFieldsStr(fields, COMMA_DELIMITER));
+ } catch (NumberFormatException e) {
+ throw new BadRequestException("flowRunId is not a numeric value.");
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("Requested Invalid Field.");
+ } catch (Exception e) {
+ LOG.error("Error getting entity", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ if (entity == null) {
+ throw new NotFoundException("Timeline entity {id: " + parseStr(entityId) +
+ ", type: " + parseStr(entityType) + " } is not found");
+ }
+ return entity;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a34844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index 45ddd1d..626c770 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
@@ -397,6 +398,10 @@ public class FileSystemTimelineReaderImpl extends AbstractService
new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
return createEntityToBeReturned(entity, fieldsToRetrieve);
+ } catch (FileNotFoundException e) {
+ LOG.info("Cannot find entity {id:" + entityId + " , type:" + entityType +
+ "}. Will send HTTP 404 in response.");
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a34844e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index a9145d0..0f7c22f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -18,25 +18,37 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
+import java.util.Set;
import javax.ws.rs.core.MediaType;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TestFileSystemTimelineReaderImpl;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
@@ -46,12 +58,23 @@ public class TestTimelineReaderWebServices {
private int serverPort;
private TimelineReaderServer server;
+ @BeforeClass
+ public static void setup() throws Exception {
+ TestFileSystemTimelineReaderImpl.setup();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TestFileSystemTimelineReaderImpl.tearDown();
+ }
+
@Before
public void init() throws Exception {
try {
Configuration config = new YarnConfiguration();
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
"localhost:0");
+ config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
server = new TimelineReaderServer();
server.init(config);
server.start();
@@ -69,6 +92,22 @@ public class TestTimelineReaderWebServices {
}
}
+ private static TimelineEntity newEntity(String type, String id) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setIdentifier(new TimelineEntity.Identifier(type, id));
+ return entity;
+ }
+
+ private static void verifyHttpResponse(Client client, URI uri,
+ Status status) {
+ ClientResponse resp =
+ client.resource(uri).accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertNotNull(resp);
+ assertTrue("Response from server should have been " + status,
+ resp.getClientResponseStatus().equals(status));
+ }
+
private static Client createClient() {
ClientConfig cfg = new DefaultClientConfig();
cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
@@ -76,14 +115,19 @@ public class TestTimelineReaderWebServices {
new DummyURLConnectionFactory()), cfg);
}
- private static ClientResponse getResponse(Client client, URI uri) throws Exception {
+ private static ClientResponse getResponse(Client client, URI uri)
+ throws Exception {
ClientResponse resp =
client.resource(uri).accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
if (resp == null ||
resp.getClientResponseStatus() != ClientResponse.Status.OK) {
- System.out.println(resp.getClientResponseStatus());
- throw new IOException("Incorrect response from timeline reader.");
+ String msg = new String();
+ if (resp != null) {
+ msg = resp.getClientResponseStatus().toString();
+ }
+ throw new IOException("Incorrect response from timeline reader. " +
+ "Status=" + msg);
}
return resp;
}
@@ -102,8 +146,7 @@ public class TestTimelineReaderWebServices {
}
@Test
- public void testAbout()
- throws IOException {
+ public void testAbout() throws Exception {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/");
Client client = createClient();
try {
@@ -111,9 +154,406 @@ public class TestTimelineReaderWebServices {
TimelineAbout about = resp.getEntity(TimelineAbout.class);
Assert.assertNotNull(about);
Assert.assertEquals("Timeline Reader API", about.getAbout());
- } catch (Exception re) {
- throw new IOException(
- "Failed to get the response from timeline reader.", re);
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntityDefaultView() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/cluster1/app1/app/id_1");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("id_1", entity.getId());
+ assertEquals("app", entity.getType());
+ assertEquals(1425016502000L, entity.getCreatedTime());
+ assertEquals(1425016503000L, entity.getModifiedTime());
+ // Default view i.e. when no fields are specified, entity contains only
+ // entity id, entity type, created and modified time.
+ assertEquals(0, entity.getConfigs().size());
+ assertEquals(0, entity.getMetrics().size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntityWithUserAndFlowInfo() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/cluster1/app1/app/id_1?userId=user1&" +
+ "flowId=flow1&flowRunId=1");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("id_1", entity.getId());
+ assertEquals("app", entity.getType());
+ assertEquals(1425016502000L, entity.getCreatedTime());
+ assertEquals(1425016503000L, entity.getModifiedTime());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntityCustomFields() throws Exception {
+ Client client = createClient();
+ try {
+ // Fields are case insensitive.
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/cluster1/app1/app/id_1?fields=CONFIGS,Metrics,info");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("id_1", entity.getId());
+ assertEquals("app", entity.getType());
+ assertEquals(3, entity.getConfigs().size());
+ assertEquals(3, entity.getMetrics().size());
+ assertEquals(1, entity.getInfo().size());
+ // No events will be returned as events are not part of fields.
+ assertEquals(0, entity.getEvents().size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntityAllFields() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/cluster1/app1/app/id_1?fields=ALL");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("id_1", entity.getId());
+ assertEquals("app", entity.getType());
+ assertEquals(3, entity.getConfigs().size());
+ assertEquals(3, entity.getMetrics().size());
+ assertEquals(1, entity.getInfo().size());
+ assertEquals(2, entity.getEvents().size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntityNotPresent() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/cluster1/app1/app/id_10");
+ verifyHttpResponse(client, uri, Status.NOT_FOUND);
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntities() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(4, entities.size());
+ assertTrue("Entities id_1, id_2, id_3 and id_4 should have been" +
+ " present in response",
+ entities.contains(newEntity("app", "id_1")) &&
+ entities.contains(newEntity("app", "id_2")) &&
+ entities.contains(newEntity("app", "id_3")) &&
+ entities.contains(newEntity("app", "id_4")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesWithLimit() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?limit=2");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ // Entities returned are based on most recent created time.
+ assertTrue("Entities with id_1 and id_4 should have been present " +
+ "in response based on entity created time.",
+ entities.contains(newEntity("app", "id_1")) &&
+ entities.contains(newEntity("app", "id_4")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?limit=3");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ // Even though 2 entities out of 4 have same created time, one entity
+ // is left out due to limit
+ assertEquals(3, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesBasedOnCreatedTime() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?createdTimeStart=1425016502030&"
+ + "createdTimeEnd=1425016502060");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_4 should have been present in response.",
+ entities.contains(newEntity("app", "id_4")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?createdTimeEnd=1425016502010");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(3, entities.size());
+ assertFalse("Entity with id_4 should not have been present in response.",
+ entities.contains(newEntity("app", "id_4")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?createdTimeStart=1425016502010");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_4 should have been present in response.",
+ entities.contains(newEntity("app", "id_4")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesBasedOnModifiedTime() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?modifiedTimeStart=1425016502090"
+ + "&modifiedTimeEnd=1425016503020");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ assertTrue("Entities with id_1 and id_4 should have been" +
+ " present in response.",
+ entities.contains(newEntity("app", "id_1")) &&
+ entities.contains(newEntity("app", "id_4")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?modifiedTimeEnd=1425016502090");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ assertTrue("Entities with id_2 and id_3 should have been " +
+ "present in response.",
+ entities.contains(newEntity("app", "id_2")) &&
+ entities.contains(newEntity("app", "id_3")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?modifiedTimeStart=1425016503005");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_4 should have been present in response.",
+ entities.contains(newEntity("app", "id_4")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByRelations() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?relatesto=flow:flow1");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_1 should have been present in response.",
+ entities.contains(newEntity("app", "id_1")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?isrelatedto=type1:tid1_2,type2:" +
+ "tid2_1%60");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_1 should have been present in response.",
+ entities.contains(newEntity("app", "id_1")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?isrelatedto=type1:tid1_1:tid1_2" +
+ ",type2:tid2_1%60");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_1 should have been present in response.",
+ entities.contains(newEntity("app", "id_1")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByConfigFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?conffilters=config_1:123," +
+ "config_3:abc");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ assertTrue("Entities with id_1 and id_3 should have been present" +
+ " in response.",
+ entities.contains(newEntity("app", "id_1")) &&
+ entities.contains(newEntity("app", "id_3")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByInfoFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?infofilters=info2:3.5");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_3 should have been present in response.",
+ entities.contains(newEntity("app", "id_3")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByMetricFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?metricfilters=metric3");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ assertTrue("Entities with id_1 and id_2 should have been present" +
+ " in response.",
+ entities.contains(newEntity("app", "id_1")) &&
+ entities.contains(newEntity("app", "id_2")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByEventFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?eventfilters=event_2,event_4");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_3 should have been present in response.",
+ entities.contains(newEntity("app", "id_3")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesNoMatch() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?metricfilters=metric7&" +
+ "isrelatedto=type1:tid1_1;tid1_2,type2:tid2_1%60&relatesto=" +
+ "flow:flow1&eventfilters=event_2,event_4&infofilters=info2:3.5" +
+ "&createdTimeStart=1425016502030&createdTimeEnd=1425016502060");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testInvalidValuesHandling() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?flowRunId=a23b");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entity/cluster1/app1/app/id_1?flowRunId=2ab15");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app/?limit=#$561av");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
} finally {
client.destroy();
}
[42/50] [abbrv] hadoop git commit: YARN-4074. [timeline reader]
implement support for querying for flows and flow runs (sjlee via vrushali)
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index c514c20..889ae19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -20,13 +20,8 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
-import java.util.NavigableMap;
-import java.util.NavigableSet;
import java.util.Set;
-import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -34,47 +29,17 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-
-import com.google.common.base.Preconditions;
public class HBaseTimelineReaderImpl
extends AbstractService implements TimelineReader {
private static final Log LOG = LogFactory
.getLog(HBaseTimelineReaderImpl.class);
- private static final long DEFAULT_BEGIN_TIME = 0L;
- private static final long DEFAULT_END_TIME = Long.MAX_VALUE;
private Configuration hbaseConf = null;
private Connection conn;
- private EntityTable entityTable;
- private AppToFlowTable appToFlowTable;
- private ApplicationTable applicationTable;
public HBaseTimelineReaderImpl() {
super(HBaseTimelineReaderImpl.class.getName());
@@ -85,9 +50,6 @@ public class HBaseTimelineReaderImpl
super.serviceInit(conf);
hbaseConf = HBaseConfiguration.create(conf);
conn = ConnectionFactory.createConnection(hbaseConf);
- entityTable = new EntityTable();
- appToFlowTable = new AppToFlowTable();
- applicationTable = new ApplicationTable();
}
@Override
@@ -104,35 +66,10 @@ public class HBaseTimelineReaderImpl
String flowId, Long flowRunId, String appId, String entityType,
String entityId, EnumSet<Field> fieldsToRetrieve)
throws IOException {
- validateParams(userId, clusterId, appId, entityType, entityId, true);
- // In reality both should be null or neither should be null
- if (flowId == null || flowRunId == null) {
- FlowContext context = lookupFlowContext(clusterId, appId);
- flowId = context.flowId;
- flowRunId = context.flowRunId;
- }
- if (fieldsToRetrieve == null) {
- fieldsToRetrieve = EnumSet.noneOf(Field.class);
- }
-
- boolean isApplication = isApplicationEntity(entityType);
- byte[] rowKey = isApplication ?
- ApplicationRowKey.getRowKey(clusterId, userId, flowId, flowRunId,
- appId) :
- EntityRowKey.getRowKey(clusterId, userId, flowId, flowRunId, appId,
- entityType, entityId);
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- Result result = isApplication ?
- applicationTable.getResult(hbaseConf, conn, get) :
- entityTable.getResult(hbaseConf, conn, get);
- return parseEntity(result, fieldsToRetrieve,
- false, DEFAULT_BEGIN_TIME, DEFAULT_END_TIME, false, DEFAULT_BEGIN_TIME,
- DEFAULT_END_TIME, null, null, null, null, null, null, isApplication);
- }
-
- private static boolean isApplicationEntity(String entityType) {
- return TimelineEntityType.YARN_APPLICATION.toString().equals(entityType);
+ TimelineEntityReader reader =
+ TimelineEntityReaderFactory.createSingleEntityReader(userId, clusterId,
+ flowId, flowRunId, appId, entityType, entityId, fieldsToRetrieve);
+ return reader.readEntity(hbaseConf, conn);
}
@Override
@@ -144,361 +81,12 @@ public class HBaseTimelineReaderImpl
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
EnumSet<Field> fieldsToRetrieve) throws IOException {
- validateParams(userId, clusterId, appId, entityType, null, false);
- // In reality both should be null or neither should be null
- if (flowId == null || flowRunId == null) {
- FlowContext context = lookupFlowContext(clusterId, appId);
- flowId = context.flowId;
- flowRunId = context.flowRunId;
- }
- if (limit == null) {
- limit = TimelineReader.DEFAULT_LIMIT;
- }
- if (createdTimeBegin == null) {
- createdTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (createdTimeEnd == null) {
- createdTimeEnd = DEFAULT_END_TIME;
- }
- if (modifiedTimeBegin == null) {
- modifiedTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (modifiedTimeEnd == null) {
- modifiedTimeEnd = DEFAULT_END_TIME;
- }
- if (fieldsToRetrieve == null) {
- fieldsToRetrieve = EnumSet.noneOf(Field.class);
- }
-
- NavigableSet<TimelineEntity> entities = new TreeSet<>();
- boolean isApplication = isApplicationEntity(entityType);
- if (isApplication) {
- // If getEntities() is called for an application, there can be at most
- // one entity. If the entity passes the filter, it is returned. Otherwise,
- // an empty set is returned.
- byte[] rowKey = ApplicationRowKey.getRowKey(clusterId, userId, flowId,
- flowRunId, appId);
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- Result result = applicationTable.getResult(hbaseConf, conn, get);
- TimelineEntity entity = parseEntity(result, fieldsToRetrieve,
- true, createdTimeBegin, createdTimeEnd, true, modifiedTimeBegin,
- modifiedTimeEnd, isRelatedTo, relatesTo, infoFilters, configFilters,
- eventFilters, metricFilters, isApplication);
- if (entity != null) {
- entities.add(entity);
- }
- } else {
- // Scan through part of the table to find the entities belong to one app
- // and one type
- Scan scan = new Scan();
- scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
- clusterId, userId, flowId, flowRunId, appId, entityType));
- scan.setMaxVersions(Integer.MAX_VALUE);
- ResultScanner scanner =
- entityTable.getResultScanner(hbaseConf, conn, scan);
- for (Result result : scanner) {
- TimelineEntity entity = parseEntity(result, fieldsToRetrieve,
- true, createdTimeBegin, createdTimeEnd,
- true, modifiedTimeBegin, modifiedTimeEnd,
- isRelatedTo, relatesTo, infoFilters, configFilters, eventFilters,
- metricFilters, isApplication);
- if (entity == null) {
- continue;
- }
- if (entities.size() > limit) {
- entities.pollLast();
- }
- entities.add(entity);
- }
- }
- return entities;
- }
-
- private FlowContext lookupFlowContext(String clusterId, String appId)
- throws IOException {
- byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
- Get get = new Get(rowKey);
- Result result = appToFlowTable.getResult(hbaseConf, conn, get);
- if (result != null && !result.isEmpty()) {
- return new FlowContext(
- AppToFlowColumn.FLOW_ID.readResult(result).toString(),
- ((Number) AppToFlowColumn.FLOW_RUN_ID.readResult(result)).longValue());
- } else {
- throw new IOException(
- "Unable to find the context flow ID and flow run ID for clusterId=" +
- clusterId + ", appId=" + appId);
- }
- }
-
- private static class FlowContext {
- private String flowId;
- private Long flowRunId;
- public FlowContext(String flowId, Long flowRunId) {
- this.flowId = flowId;
- this.flowRunId = flowRunId;
- }
- }
-
- private static void validateParams(String userId, String clusterId,
- String appId, String entityType, String entityId, boolean checkEntityId) {
- Preconditions.checkNotNull(userId, "userId shouldn't be null");
- Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
- Preconditions.checkNotNull(appId, "appId shouldn't be null");
- Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
- if (checkEntityId) {
- Preconditions.checkNotNull(entityId, "entityId shouldn't be null");
- }
- }
-
- private static TimelineEntity parseEntity(
- Result result, EnumSet<Field> fieldsToRetrieve,
- boolean checkCreatedTime, long createdTimeBegin, long createdTimeEnd,
- boolean checkModifiedTime, long modifiedTimeBegin, long modifiedTimeEnd,
- Map<String, Set<String>> isRelatedTo, Map<String, Set<String>> relatesTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> eventFilters, Set<String> metricFilters,
- boolean isApplication)
- throws IOException {
- if (result == null || result.isEmpty()) {
- return null;
- }
- TimelineEntity entity = new TimelineEntity();
- String entityType = isApplication ?
- TimelineEntityType.YARN_APPLICATION.toString() :
- EntityColumn.TYPE.readResult(result).toString();
- entity.setType(entityType);
- String entityId = isApplication ?
- ApplicationColumn.ID.readResult(result).toString() :
- EntityColumn.ID.readResult(result).toString();
- entity.setId(entityId);
-
- // fetch created time
- Number createdTime = isApplication ?
- (Number)ApplicationColumn.CREATED_TIME.readResult(result) :
- (Number)EntityColumn.CREATED_TIME.readResult(result);
- entity.setCreatedTime(createdTime.longValue());
- if (checkCreatedTime && (entity.getCreatedTime() < createdTimeBegin ||
- entity.getCreatedTime() > createdTimeEnd)) {
- return null;
- }
-
- // fetch modified time
- Number modifiedTime = isApplication ?
- (Number)ApplicationColumn.MODIFIED_TIME.readResult(result) :
- (Number)EntityColumn.MODIFIED_TIME.readResult(result);
- entity.setModifiedTime(modifiedTime.longValue());
- if (checkModifiedTime && (entity.getModifiedTime() < modifiedTimeBegin ||
- entity.getModifiedTime() > modifiedTimeEnd)) {
- return null;
- }
-
- // fetch is related to entities
- boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
- if (isApplication) {
- readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
- true);
- } else {
- readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO,
- true);
- }
- if (checkIsRelatedTo && !TimelineReaderUtils.matchRelations(
- entity.getIsRelatedToEntities(), isRelatedTo)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
- entity.getIsRelatedToEntities().clear();
- }
- }
-
- // fetch relates to entities
- boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
- if (isApplication) {
- readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
- false);
- } else {
- readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
- }
- if (checkRelatesTo && !TimelineReaderUtils.matchRelations(
- entity.getRelatesToEntities(), relatesTo)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.RELATES_TO)) {
- entity.getRelatesToEntities().clear();
- }
- }
-
- // fetch info
- boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
- if (isApplication) {
- readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
- } else {
- readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
- }
- if (checkInfo &&
- !TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.INFO)) {
- entity.getInfo().clear();
- }
- }
-
- // fetch configs
- boolean checkConfigs = configFilters != null && configFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
- if (isApplication) {
- readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
- } else {
- readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
- }
- if (checkConfigs && !TimelineReaderUtils.matchFilters(
- entity.getConfigs(), configFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.CONFIGS)) {
- entity.getConfigs().clear();
- }
- }
-
- // fetch events
- boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
- readEvents(entity, result, isApplication);
- if (checkEvents && !TimelineReaderUtils.matchEventFilters(
- entity.getEvents(), eventFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.EVENTS)) {
- entity.getEvents().clear();
- }
- }
-
- // fetch metrics
- boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
- readMetrics(entity, result, isApplication);
- if (checkMetrics && !TimelineReaderUtils.matchMetricFilters(
- entity.getMetrics(), metricFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.METRICS)) {
- entity.getMetrics().clear();
- }
- }
- return entity;
- }
-
- private static <T> void readRelationship(
- TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
- boolean isRelatedTo) throws IOException {
- // isRelatedTo and relatesTo are of type Map<String, Set<String>>
- Map<String, Object> columns = prefix.readResults(result);
- for (Map.Entry<String, Object> column : columns.entrySet()) {
- for (String id : Separator.VALUES.splitEncoded(
- column.getValue().toString())) {
- if (isRelatedTo) {
- entity.addIsRelatedToEntity(column.getKey(), id);
- } else {
- entity.addRelatesToEntity(column.getKey(), id);
- }
- }
- }
- }
-
- private static <T> void readKeyValuePairs(
- TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
- boolean isConfig) throws IOException {
- // info and configuration are of type Map<String, Object or String>
- Map<String, Object> columns = prefix.readResults(result);
- if (isConfig) {
- for (Map.Entry<String, Object> column : columns.entrySet()) {
- entity.addConfig(column.getKey(), column.getValue().toString());
- }
- } else {
- entity.addInfo(columns);
- }
- }
-
- /**
- * Read events from the entity table or the application table. The column name
- * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
- * if there is no info associated with the event.
- *
- * See {@link EntityTable} and {@link ApplicationTable} for a more detailed
- * schema description.
- */
- private static void readEvents(TimelineEntity entity, Result result,
- boolean isApplication) throws IOException {
- Map<String, TimelineEvent> eventsMap = new HashMap<>();
- Map<?, Object> eventsResult = isApplication ?
- ApplicationColumnPrefix.EVENT.
- readResultsHavingCompoundColumnQualifiers(result) :
- EntityColumnPrefix.EVENT.
- readResultsHavingCompoundColumnQualifiers(result);
- for (Map.Entry<?, Object> eventResult : eventsResult.entrySet()) {
- byte[][] karr = (byte[][])eventResult.getKey();
- // the column name is of the form "eventId=timestamp=infoKey"
- if (karr.length == 3) {
- String id = Bytes.toString(karr[0]);
- long ts = TimelineWriterUtils.invert(Bytes.toLong(karr[1]));
- String key = Separator.VALUES.joinEncoded(id, Long.toString(ts));
- TimelineEvent event = eventsMap.get(key);
- if (event == null) {
- event = new TimelineEvent();
- event.setId(id);
- event.setTimestamp(ts);
- eventsMap.put(key, event);
- }
- // handle empty info
- String infoKey = karr[2].length == 0 ? null : Bytes.toString(karr[2]);
- if (infoKey != null) {
- event.addInfo(infoKey, eventResult.getValue());
- }
- } else {
- LOG.warn("incorrectly formatted column name: it will be discarded");
- continue;
- }
- }
- Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
- entity.addEvents(eventsSet);
- }
-
- private static void readMetrics(TimelineEntity entity, Result result,
- boolean isApplication) throws IOException {
- NavigableMap<String, NavigableMap<Long, Number>> metricsResult;
- if (isApplication) {
- metricsResult =
- ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
- } else {
- metricsResult =
- EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
- }
- for (Map.Entry<String, NavigableMap<Long, Number>> metricResult:
- metricsResult.entrySet()) {
- TimelineMetric metric = new TimelineMetric();
- metric.setId(metricResult.getKey());
- // Simply assume that if the value set contains more than 1 elements, the
- // metric is a TIME_SERIES metric, otherwise, it's a SINGLE_VALUE metric
- metric.setType(metricResult.getValue().size() > 1 ?
- TimelineMetric.Type.TIME_SERIES : TimelineMetric.Type.SINGLE_VALUE);
- metric.addValues(metricResult.getValue());
- entity.addMetric(metric);
- }
+ TimelineEntityReader reader =
+ TimelineEntityReaderFactory.createMultipleEntitiesReader(userId,
+ clusterId, flowId, flowRunId, appId, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin,
+ modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters,
+ metricFilters, eventFilters, fieldsToRetrieve);
+ return reader.readEntities(hbaseConf, conn);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
new file mode 100644
index 0000000..0d1134c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+
+/**
+ * The base class for reading and deserializing timeline entities from the
+ * HBase storage. Different types can be defined for different types of the
+ * entities that are being requested.
+ */
+abstract class TimelineEntityReader {
+ protected final boolean singleEntityRead;
+
+ protected String userId;
+ protected String clusterId;
+ protected String flowId;
+ protected Long flowRunId;
+ protected String appId;
+ protected String entityType;
+ protected EnumSet<Field> fieldsToRetrieve;
+ // used only for a single entity read mode
+ protected String entityId;
+ // used only for multiple entity read mode
+ protected Long limit;
+ protected Long createdTimeBegin;
+ protected Long createdTimeEnd;
+ protected Long modifiedTimeBegin;
+ protected Long modifiedTimeEnd;
+ protected Map<String, Set<String>> relatesTo;
+ protected Map<String, Set<String>> isRelatedTo;
+ protected Map<String, Object> infoFilters;
+ protected Map<String, String> configFilters;
+ protected Set<String> metricFilters;
+ protected Set<String> eventFilters;
+
+ /**
+ * Main table the entity reader uses.
+ */
+ protected BaseTable<?> table;
+
+ /**
+ * Instantiates a reader for multiple-entity reads.
+ */
+ protected TimelineEntityReader(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) {
+ this.singleEntityRead = false;
+ this.userId = userId;
+ this.clusterId = clusterId;
+ this.flowId = flowId;
+ this.flowRunId = flowRunId;
+ this.appId = appId;
+ this.entityType = entityType;
+ this.fieldsToRetrieve = fieldsToRetrieve;
+ this.limit = limit;
+ this.createdTimeBegin = createdTimeBegin;
+ this.createdTimeEnd = createdTimeEnd;
+ this.modifiedTimeBegin = modifiedTimeBegin;
+ this.modifiedTimeEnd = modifiedTimeEnd;
+ this.relatesTo = relatesTo;
+ this.isRelatedTo = isRelatedTo;
+ this.infoFilters = infoFilters;
+ this.configFilters = configFilters;
+ this.metricFilters = metricFilters;
+ this.eventFilters = eventFilters;
+
+ this.table = getTable();
+ }
+
+ /**
+ * Instantiates a reader for single-entity reads.
+ */
+ protected TimelineEntityReader(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fieldsToRetrieve) {
+ this.singleEntityRead = true;
+ this.userId = userId;
+ this.clusterId = clusterId;
+ this.flowId = flowId;
+ this.flowRunId = flowRunId;
+ this.appId = appId;
+ this.entityType = entityType;
+ this.fieldsToRetrieve = fieldsToRetrieve;
+ this.entityId = entityId;
+
+ this.table = getTable();
+ }
+
+ /**
+ * Reads and deserializes a single timeline entity from the HBase storage.
+ */
+ public TimelineEntity readEntity(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ validateParams();
+ augmentParams(hbaseConf, conn);
+
+ Result result = getResult(hbaseConf, conn);
+ return parseEntity(result);
+ }
+
+ /**
+ * Reads and deserializes a set of timeline entities from the HBase storage.
+ * It goes through all the results available, and returns the number of
+ * entries as specified in the limit in the entity's natural sort order.
+ */
+ public Set<TimelineEntity> readEntities(Configuration hbaseConf,
+ Connection conn) throws IOException {
+ validateParams();
+ augmentParams(hbaseConf, conn);
+
+ NavigableSet<TimelineEntity> entities = new TreeSet<>();
+ Iterable<Result> results = getResults(hbaseConf, conn);
+ for (Result result : results) {
+ TimelineEntity entity = parseEntity(result);
+ if (entity == null) {
+ continue;
+ }
+ entities.add(entity);
+ if (entities.size() > limit) {
+ entities.pollLast();
+ }
+ }
+ return entities;
+ }
+
+ /**
+ * Returns the main table to be used by the entity reader.
+ */
+ protected abstract BaseTable<?> getTable();
+
+ /**
+ * Validates the required parameters to read the entities.
+ */
+ protected abstract void validateParams();
+
+ /**
+ * Sets certain parameters to defaults if the values are not provided.
+ */
+ protected abstract void augmentParams(Configuration hbaseConf,
+ Connection conn) throws IOException;
+
+ /**
+ * Fetches a {@link Result} instance for a single-entity read.
+ *
+ * @return the {@link Result} instance or null if no such record is found.
+ */
+ protected abstract Result getResult(Configuration hbaseConf, Connection conn)
+ throws IOException;
+
+ /**
+ * Fetches an iterator for {@link Result} instances for a multi-entity read.
+ */
+ protected abstract Iterable<Result> getResults(Configuration hbaseConf,
+ Connection conn) throws IOException;
+
+ /**
+ * Given a {@link Result} instance, deserializes and creates a
+ * {@link TimelineEntity}.
+ *
+ * @return the {@link TimelineEntity} instance, or null if the {@link Result}
+ * is null or empty.
+ */
+ protected abstract TimelineEntity parseEntity(Result result)
+ throws IOException;
+
+ /**
+ * Helper method for reading and deserializing {@link TimelineMetric} objects
+ * using the specified column prefix. The timeline metrics then are added to
+ * the given timeline entity.
+ */
+ protected void readMetrics(TimelineEntity entity, Result result,
+ ColumnPrefix<?> columnPrefix) throws IOException {
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ columnPrefix.readResultsWithTimestamps(result);
+ for (Map.Entry<String, NavigableMap<Long, Number>> metricResult:
+ metricsResult.entrySet()) {
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId(metricResult.getKey());
+ // Simply assume that if the value set contains more than 1 elements, the
+ // metric is a TIME_SERIES metric, otherwise, it's a SINGLE_VALUE metric
+ metric.setType(metricResult.getValue().size() > 1 ?
+ TimelineMetric.Type.TIME_SERIES : TimelineMetric.Type.SINGLE_VALUE);
+ metric.addValues(metricResult.getValue());
+ entity.addMetric(metric);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
new file mode 100644
index 0000000..4fdef40
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+
+/**
+ * Factory methods for instantiating a timeline entity reader.
+ */
+class TimelineEntityReaderFactory {
+ /**
+ * Creates a timeline entity reader instance for reading a single entity with
+ * the specified input.
+ */
+ public static TimelineEntityReader createSingleEntityReader(String userId,
+ String clusterId, String flowId, Long flowRunId, String appId,
+ String entityType, String entityId, EnumSet<Field> fieldsToRetrieve) {
+ // currently the types that are handled separate from the generic entity
+ // table are application, flow run, and flow activity entities
+ if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
+ return new ApplicationEntityReader(userId, clusterId, flowId, flowRunId,
+ appId, entityType, entityId, fieldsToRetrieve);
+ } else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
+ return new FlowRunEntityReader(userId, clusterId, flowId, flowRunId,
+ appId, entityType, entityId, fieldsToRetrieve);
+ } else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
+ return new FlowActivityEntityReader(userId, clusterId, flowId, flowRunId,
+ appId, entityType, entityId, fieldsToRetrieve);
+ } else {
+ // assume we're dealing with a generic entity read
+ return new GenericEntityReader(userId, clusterId, flowId, flowRunId,
+ appId, entityType, entityId, fieldsToRetrieve);
+ }
+ }
+
+ /**
+ * Creates a timeline entity reader instance for reading set of entities with
+ * the specified input and predicates.
+ */
+ public static TimelineEntityReader createMultipleEntitiesReader(String userId,
+ String clusterId, String flowId, Long flowRunId, String appId,
+ String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) {
+ // currently the types that are handled separate from the generic entity
+ // table are application, flow run, and flow activity entities
+ if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
+ return new ApplicationEntityReader(userId, clusterId, flowId, flowRunId,
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd,
+ modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
+ infoFilters, configFilters, metricFilters, eventFilters,
+ fieldsToRetrieve);
+ } else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
+ return new FlowActivityEntityReader(userId, clusterId, flowId, flowRunId,
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd,
+ modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
+ infoFilters, configFilters, metricFilters, eventFilters,
+ fieldsToRetrieve);
+ } else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
+ return new FlowRunEntityReader(userId, clusterId, flowId, flowRunId,
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd,
+ modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
+ infoFilters, configFilters, metricFilters, eventFilters,
+ fieldsToRetrieve);
+ } else {
+ // assume we're dealing with a generic entity read
+ return new GenericEntityReader(userId, clusterId, flowId, flowRunId,
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd,
+ modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
+ infoFilters, configFilters, metricFilters, eventFilters,
+ fieldsToRetrieve);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
index 5f3868b..e3b5a87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
@@ -19,14 +19,46 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.application;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
/**
* Represents a rowkey for the application table.
*/
public class ApplicationRowKey {
- // TODO: more methods are needed for this class.
+ private final String clusterId;
+ private final String userId;
+ private final String flowId;
+ private final long flowRunId;
+ private final String appId;
- // TODO: API needs to be cleaned up.
+ public ApplicationRowKey(String clusterId, String userId, String flowId,
+ long flowRunId, String appId) {
+ this.clusterId = clusterId;
+ this.userId = userId;
+ this.flowId = flowId;
+ this.flowRunId = flowRunId;
+ this.appId = appId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public String getFlowId() {
+ return flowId;
+ }
+
+ public long getFlowRunId() {
+ return flowRunId;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
/**
* Constructs a row key for the application table as follows:
@@ -46,22 +78,32 @@ public class ApplicationRowKey {
flowId));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
- byte[] second = Bytes.toBytes(ApplicationRowKey.invert(flowRunId));
+ byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
byte[] third = Bytes.toBytes(appId);
return Separator.QUALIFIERS.join(first, second, third);
}
/**
- * Converts a timestamp into its inverse timestamp to be used in (row) keys
- * where we want to have the most recent timestamp in the top of the table
- * (scans start at the most recent timestamp first).
- *
- * @param key value to be inverted so that the latest version will be first in
- * a scan.
- * @return inverted long
+ * Given the raw row key as bytes, returns the row key as an object.
*/
- public static long invert(Long key) {
- return Long.MAX_VALUE - key;
- }
+ public static ApplicationRowKey parseRowKey(byte[] rowKey) {
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
+
+ if (rowKeyComponents.length < 5) {
+ throw new IllegalArgumentException("the row key is not valid for " +
+ "an application");
+ }
+ String clusterId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[0]));
+ String userId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[1]));
+ String flowId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[2]));
+ long flowRunId =
+ TimelineWriterUtils.invert(Bytes.toLong(rowKeyComponents[3]));
+ String appId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[4]));
+ return new ApplicationRowKey(clusterId, userId, flowId, flowRunId, appId);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
index ad4fec6..ca88056 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
@@ -24,6 +24,22 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
* Represents a rowkey for the app_flow table.
*/
public class AppToFlowRowKey {
+ private final String clusterId;
+ private final String appId;
+
+ public AppToFlowRowKey(String clusterId, String appId) {
+ this.clusterId = clusterId;
+ this.appId = appId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
/**
* Constructs a row key prefix for the app_flow table as follows:
* {@code clusterId!AppId}
@@ -36,4 +52,19 @@ public class AppToFlowRowKey {
return Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId, appId));
}
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ */
+ public static AppToFlowRowKey parseRowKey(byte[] rowKey) {
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
+
+ if (rowKeyComponents.length < 2) {
+ throw new IllegalArgumentException("the row key is not valid for " +
+ "the app-to-flow table");
+ }
+
+ String clusterId = Bytes.toString(rowKeyComponents[0]);
+ String appId = Bytes.toString(rowKeyComponents[1]);
+ return new AppToFlowRowKey(clusterId, appId);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
index abba79a..9545438 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
/**
- * Implements behavior common to tables used in the timeline service storage.
+ * Implements behavior common to tables used in the timeline service storage. It
+ * is thread-safe, and can be used by multiple threads concurrently.
*
* @param <T> reference to the table instance class itself for type safety.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
index 9a72be0..6a534ed73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
@@ -26,9 +25,52 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWrit
* Represents a rowkey for the entity table.
*/
public class EntityRowKey {
- // TODO: more methods are needed for this class.
+ private final String clusterId;
+ private final String userId;
+ private final String flowId;
+ private final long flowRunId;
+ private final String appId;
+ private final String entityType;
+ private final String entityId;
- // TODO: API needs to be cleaned up.
+ public EntityRowKey(String clusterId, String userId, String flowId,
+ long flowRunId, String appId, String entityType, String entityId) {
+ this.clusterId = clusterId;
+ this.userId = userId;
+ this.flowId = flowId;
+ this.flowRunId = flowRunId;
+ this.appId = appId;
+ this.entityType = entityType;
+ this.entityId = entityId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public String getFlowId() {
+ return flowId;
+ }
+
+ public long getFlowRunId() {
+ return flowRunId;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ public String getEntityType() {
+ return entityType;
+ }
+
+ public String getEntityId() {
+ return entityId;
+ }
/**
* Constructs a row key prefix for the entity table as follows:
@@ -106,4 +148,32 @@ public class EntityRowKey {
return Separator.QUALIFIERS.join(first, second, third);
}
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ */
+ public static EntityRowKey parseRowKey(byte[] rowKey) {
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
+
+ if (rowKeyComponents.length < 7) {
+ throw new IllegalArgumentException("the row key is not valid for " +
+ "an entity");
+ }
+
+ String userId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[0]));
+ String clusterId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[1]));
+ String flowId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[2]));
+ long flowRunId =
+ TimelineWriterUtils.invert(Bytes.toLong(rowKeyComponents[3]));
+ String appId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[4]));
+ String entityType =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[5]));
+ String entityId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[6]));
+ return new EntityRowKey(clusterId, userId, flowId, flowRunId, appId,
+ entityType, entityId);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
index 19e4e83..18ca599 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
@@ -55,6 +55,10 @@ public class FlowActivityRowKey {
return flowId;
}
+ public static byte[] getRowKeyPrefix(String clusterId) {
+ return Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId, ""));
+ }
+
/**
* Constructs a row key for the flow activity table as follows:
* {@code clusterId!dayTimestamp!user!flowId}
@@ -65,7 +69,8 @@ public class FlowActivityRowKey {
* @param flowId
* @return byte array with the row key prefix
*/
- public static byte[] getRowKey(String clusterId, String userId, String flowId) {
+ public static byte[] getRowKey(String clusterId, String userId,
+ String flowId) {
long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
.currentTimeMillis());
return getRowKey(clusterId, dayTs, userId, flowId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
index e133241..880d481 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
@@ -25,7 +25,34 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWrit
* Represents a rowkey for the flow run table.
*/
public class FlowRunRowKey {
- // TODO: more methods are needed for this class like parse row key
+ private final String clusterId;
+ private final String userId;
+ private final String flowId;
+ private final long flowRunId;
+
+ public FlowRunRowKey(String clusterId, String userId, String flowId,
+ long flowRunId) {
+ this.clusterId = clusterId;
+ this.userId = userId;
+ this.flowId = flowId;
+ this.flowRunId = flowRunId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public String getFlowId() {
+ return flowId;
+ }
+
+ public long getFlowRunId() {
+ return flowRunId;
+ }
/**
* Constructs a row key for the entity table as follows: {
@@ -47,4 +74,25 @@ public class FlowRunRowKey {
return Separator.QUALIFIERS.join(first, second);
}
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ */
+ public static FlowRunRowKey parseRowKey(byte[] rowKey) {
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
+
+ if (rowKeyComponents.length < 4) {
+ throw new IllegalArgumentException("the row key is not valid for " +
+ "a flow run");
+ }
+
+ String clusterId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[0]));
+ String userId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[1]));
+ String flowId =
+ Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[2]));
+ long flowRunId =
+ TimelineWriterUtils.invert(Bytes.toLong(rowKeyComponents[3]));
+ return new FlowRunRowKey(clusterId, userId, flowId, flowRunId);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
index a1948aa..651bb3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
@@ -18,6 +18,15 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Cell;
@@ -33,15 +42,6 @@ import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
/**
* Invoked via the coprocessor when a Get or a Scan is issued for flow run
* table. Looks through the list of cells per row, checks their tags and does
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index 3962341..01920b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -508,32 +508,28 @@ public class TestHBaseTimelineStorage {
private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
String flow, long runid, String appName, TimelineEntity te) {
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
-
- assertTrue(rowKeyComponents.length == 7);
- assertEquals(user, Bytes.toString(rowKeyComponents[0]));
- assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
- assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.invert(runid),
- Bytes.toLong(rowKeyComponents[3]));
- assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
- assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
- assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
+ EntityRowKey key = EntityRowKey.parseRowKey(rowKey);
+
+ assertEquals(user, key.getUserId());
+ assertEquals(cluster, key.getClusterId());
+ assertEquals(flow, key.getFlowId());
+ assertEquals(runid, key.getFlowRunId());
+ assertEquals(appName, key.getAppId());
+ assertEquals(te.getType(), key.getEntityType());
+ assertEquals(te.getId(), key.getEntityId());
return true;
}
private boolean isApplicationRowKeyCorrect(byte[] rowKey, String cluster,
String user, String flow, long runid, String appName) {
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
+ ApplicationRowKey key = ApplicationRowKey.parseRowKey(rowKey);
- assertTrue(rowKeyComponents.length == 5);
- assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
- assertEquals(user, Bytes.toString(rowKeyComponents[1]));
- assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.invert(runid),
- Bytes.toLong(rowKeyComponents[3]));
- assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
+ assertEquals(cluster, key.getClusterId());
+ assertEquals(user, key.getUserId());
+ assertEquals(flow, key.getFlowId());
+ assertEquals(runid, key.getFlowRunId());
+ assertEquals(appName, key.getAppId());
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
index f8331fa..d18613a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
@@ -45,7 +45,7 @@ class TestFlowDataGenerator {
String type = TimelineEntityType.YARN_APPLICATION.toString();
entity.setId(id);
entity.setType(type);
- Long cTime = 1425016501000L;
+ long cTime = 1425016501000L;
entity.setCreatedTime(cTime);
// add metrics
@@ -54,8 +54,8 @@ class TestFlowDataGenerator {
m1.setId(metric1);
Map<Long, Number> metricValues = new HashMap<Long, Number>();
long ts = System.currentTimeMillis();
- metricValues.put(ts - 100000, 2);
- metricValues.put(ts - 80000, 40);
+ metricValues.put(ts - 100000, 2L);
+ metricValues.put(ts - 80000, 40L);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
@@ -64,8 +64,8 @@ class TestFlowDataGenerator {
m2.setId(metric2);
metricValues = new HashMap<Long, Number>();
ts = System.currentTimeMillis();
- metricValues.put(ts - 100000, 31);
- metricValues.put(ts - 80000, 57);
+ metricValues.put(ts - 100000, 31L);
+ metricValues.put(ts - 80000, 57L);
m2.setType(Type.TIME_SERIES);
m2.setValues(metricValues);
metrics.add(m2);
@@ -80,7 +80,7 @@ class TestFlowDataGenerator {
String type = TimelineEntityType.YARN_APPLICATION.toString();
entity.setId(id);
entity.setType(type);
- Long cTime = 1425016501000L;
+ long cTime = 1425016501000L;
entity.setCreatedTime(cTime);
// add metrics
Set<TimelineMetric> metrics = new HashSet<>();
@@ -103,8 +103,8 @@ class TestFlowDataGenerator {
String type = TimelineEntityType.YARN_APPLICATION.toString();
entity.setId(id);
entity.setType(type);
- Long cTime = 20000000000000L;
- Long mTime = 1425026901000L;
+ long cTime = 20000000000000L;
+ long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
entity.setModifiedTime(mTime);
// add metrics
@@ -113,10 +113,10 @@ class TestFlowDataGenerator {
m1.setId(metric1);
Map<Long, Number> metricValues = new HashMap<Long, Number>();
long ts = System.currentTimeMillis();
- metricValues.put(ts - 120000, 100000000);
- metricValues.put(ts - 100000, 200000000);
- metricValues.put(ts - 80000, 300000000);
- metricValues.put(ts - 60000, 400000000);
+ metricValues.put(ts - 120000, 100000000L);
+ metricValues.put(ts - 100000, 200000000L);
+ metricValues.put(ts - 80000, 300000000L);
+ metricValues.put(ts - 60000, 400000000L);
metricValues.put(ts - 40000, 50000000000L);
metricValues.put(ts - 20000, 60000000000L);
m1.setType(Type.TIME_SERIES);
@@ -126,7 +126,7 @@ class TestFlowDataGenerator {
TimelineEvent event = new TimelineEvent();
event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- Long expTs = 1436512802000L;
+ long expTs = 1436512802000L;
event.setTimestamp(expTs);
String expKey = "foo_event";
Object expVal = "test";
@@ -142,9 +142,9 @@ class TestFlowDataGenerator {
return entity;
}
- static TimelineEntity getEntityGreaterStartTime() {
+ static TimelineEntity getEntityGreaterStartTime(long startTs) {
TimelineEntity entity = new TimelineEntity();
- entity.setCreatedTime(30000000000000L);
+ entity.setCreatedTime(startTs);
entity.setId("flowRunHello with greater start time");
String type = TimelineEntityType.YARN_APPLICATION.toString();
entity.setType(type);
@@ -173,14 +173,13 @@ class TestFlowDataGenerator {
return entity;
}
- static TimelineEntity getEntityMinStartTime() {
+ static TimelineEntity getEntityMinStartTime(long startTs) {
TimelineEntity entity = new TimelineEntity();
String id = "flowRunHelloMInStartTime";
String type = TimelineEntityType.YARN_APPLICATION.toString();
entity.setId(id);
entity.setType(type);
- Long cTime = 10000000000000L;
- entity.setCreatedTime(cTime);
+ entity.setCreatedTime(startTs);
TimelineEvent event = new TimelineEvent();
event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
event.setTimestamp(System.currentTimeMillis());
@@ -195,12 +194,12 @@ class TestFlowDataGenerator {
String type = TimelineEntityType.YARN_APPLICATION.toString();
entity.setId(id);
entity.setType(type);
- Long cTime = 1425016501000L;
+ long cTime = 1425016501000L;
entity.setCreatedTime(cTime);
TimelineEvent event = new TimelineEvent();
event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- Long expTs = 1436512802000L;
+ long expTs = 1436512802000L;
event.setTimestamp(expTs);
String expKey = "foo_event";
Object expVal = "test";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index b4a0c74..6bdec6b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -21,19 +21,16 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
+import java.util.NavigableSet;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get;
@@ -42,26 +39,17 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
-import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -119,11 +107,13 @@ public class TestHBaseStorageFlowActivity {
String user = "testWriteFlowRunMinMaxToHBase_user1";
String flow = "testing_flowRun_flow_name";
String flowVersion = "CF7022C10F1354";
- Long runid = 1002345678919L;
+ long runid = 1002345678919L;
String appName = "application_100000000000_1111";
+ long minStartTs = 10000000000000L;
+ long greaterStartTs = 30000000000000L;
long endTs = 1439750690000L;
TimelineEntity entityMinStartTime = TestFlowDataGenerator
- .getEntityMinStartTime();
+ .getEntityMinStartTime(minStartTs);
try {
hbi = new HBaseTimelineWriterImpl(c1);
@@ -146,7 +136,7 @@ public class TestHBaseStorageFlowActivity {
// writer another entity with greater start time
TimelineEntity entityGreaterStartTime = TestFlowDataGenerator
- .getEntityGreaterStartTime();
+ .getEntityGreaterStartTime(greaterStartTs);
te = new TimelineEntities();
te.addEntity(entityGreaterStartTime);
appName = "application_1000000000000000_2222";
@@ -181,6 +171,31 @@ public class TestHBaseStorageFlowActivity {
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
assertEquals(1, values.size());
checkFlowActivityRunId(runid, flowVersion, values);
+
+ // use the timeline reader to verify data
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ // get the flow activity entity
+ Set<TimelineEntity> entities =
+ hbr.getEntities(null, cluster, null, null, null,
+ TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), 10L, null, null,
+ null, null, null, null, null, null, null, null, null);
+ assertEquals(1, entities.size());
+ for (TimelineEntity e : entities) {
+ FlowActivityEntity flowActivity = (FlowActivityEntity)e;
+ assertEquals(cluster, flowActivity.getCluster());
+ assertEquals(user, flowActivity.getUser());
+ assertEquals(flow, flowActivity.getFlowName());
+ assertEquals(dayTs, flowActivity.getDate().getTime());
+ Set<FlowRunEntity> flowRuns = flowActivity.getFlowRuns();
+ assertEquals(1, flowRuns.size());
+ }
+ } finally {
+ hbr.close();
+ }
}
/**
@@ -193,7 +208,7 @@ public class TestHBaseStorageFlowActivity {
String user = "testWriteFlowActivityOneFlow_user1";
String flow = "flow_activity_test_flow_name";
String flowVersion = "A122110F135BC4";
- Long runid = 1001111178919L;
+ long runid = 1001111178919L;
TimelineEntities te = new TimelineEntities();
TimelineEntity entityApp1 = TestFlowDataGenerator.getFlowApp1();
@@ -212,10 +227,35 @@ public class TestHBaseStorageFlowActivity {
}
// check flow activity
checkFlowActivityTable(cluster, user, flow, flowVersion, runid, c1);
+
+ // use the reader to verify the data
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+
+ Set<TimelineEntity> entities =
+ hbr.getEntities(user, cluster, flow, null, null,
+ TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), 10L, null, null,
+ null, null, null, null, null, null, null, null, null);
+ assertEquals(1, entities.size());
+ for (TimelineEntity e : entities) {
+ FlowActivityEntity entity = (FlowActivityEntity)e;
+ NavigableSet<FlowRunEntity> flowRuns = entity.getFlowRuns();
+ assertEquals(1, flowRuns.size());
+ for (FlowRunEntity flowRun : flowRuns) {
+ assertEquals(runid, flowRun.getRunId());
+ assertEquals(flowVersion, flowRun.getVersion());
+ }
+ }
+ } finally {
+ hbr.close();
+ }
}
private void checkFlowActivityTable(String cluster, String user, String flow,
- String flowVersion, Long runid, Configuration c1) throws IOException {
+ String flowVersion, long runid, Configuration c1) throws IOException {
Scan s = new Scan();
s.addFamily(FlowActivityColumnFamily.INFO.getBytes());
byte[] startRow = FlowActivityRowKey.getRowKey(cluster, user, flow);
@@ -263,7 +303,7 @@ public class TestHBaseStorageFlowActivity {
String user = "testManyRunsFlowActivity_c_user1";
String flow = "flow_activity_test_flow_name";
String flowVersion1 = "A122110F135BC4";
- Long runid1 = 11111111111L;
+ long runid1 = 11111111111L;
String flowVersion2 = "A12222222222C4";
long runid2 = 2222222222222L;
@@ -303,11 +343,50 @@ public class TestHBaseStorageFlowActivity {
checkFlowActivityTableSeveralRuns(cluster, user, flow, c1, flowVersion1,
runid1, flowVersion2, runid2, flowVersion3, runid3);
+ // use the timeline reader to verify data
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+
+ Set<TimelineEntity> entities =
+ hbr.getEntities(null, cluster, null, null, null,
+ TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), 10L, null, null,
+ null, null, null, null, null, null, null, null, null);
+ assertEquals(1, entities.size());
+ for (TimelineEntity e : entities) {
+ FlowActivityEntity flowActivity = (FlowActivityEntity)e;
+ assertEquals(cluster, flowActivity.getCluster());
+ assertEquals(user, flowActivity.getUser());
+ assertEquals(flow, flowActivity.getFlowName());
+ long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
+ .currentTimeMillis());
+ assertEquals(dayTs, flowActivity.getDate().getTime());
+ Set<FlowRunEntity> flowRuns = flowActivity.getFlowRuns();
+ assertEquals(3, flowRuns.size());
+ for (FlowRunEntity flowRun : flowRuns) {
+ long runId = flowRun.getRunId();
+ String version = flowRun.getVersion();
+ if (runId == runid1) {
+ assertEquals(flowVersion1, version);
+ } else if (runId == runid2) {
+ assertEquals(flowVersion2, version);
+ } else if (runId == runid3) {
+ assertEquals(flowVersion3, version);
+ } else {
+ fail("unknown run id: " + runId);
+ }
+ }
+ }
+ } finally {
+ hbr.close();
+ }
}
private void checkFlowActivityTableSeveralRuns(String cluster, String user,
- String flow, Configuration c1, String flowVersion1, Long runid1,
- String flowVersion2, Long runid2, String flowVersion3, Long runid3)
+ String flow, Configuration c1, String flowVersion1, long runid1,
+ String flowVersion2, long runid2, String flowVersion3, long runid3)
throws IOException {
Scan s = new Scan();
s.addFamily(FlowActivityColumnFamily.INFO.getBytes());
@@ -351,7 +430,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(1, rowCount);
}
- private void checkFlowActivityRunId(Long runid, String flowVersion,
+ private void checkFlowActivityRunId(long runid, String flowVersion,
Map<byte[], byte[]> values) throws IOException {
byte[] rq = ColumnHelper.getColumnQualifier(
FlowActivityColumnPrefix.RUN_ID.getColumnPrefixBytes(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index bf524ea..b0f83b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -21,20 +21,15 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get;
@@ -42,32 +37,16 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
-import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -125,11 +104,13 @@ public class TestHBaseStorageFlowRun {
String user = "testWriteFlowRunMinMaxToHBase_user1";
String flow = "testing_flowRun_flow_name";
String flowVersion = "CF7022C10F1354";
- Long runid = 1002345678919L;
+ long runid = 1002345678919L;
String appName = "application_100000000000_1111";
+ long minStartTs = 10000000000000L;
+ long greaterStartTs = 30000000000000L;
long endTs = 1439750690000L;
TimelineEntity entityMinStartTime = TestFlowDataGenerator
- .getEntityMinStartTime();
+ .getEntityMinStartTime(minStartTs);
try {
hbi = new HBaseTimelineWriterImpl(c1);
@@ -152,7 +133,7 @@ public class TestHBaseStorageFlowRun {
// writer another entity with greater start time
TimelineEntity entityGreaterStartTime = TestFlowDataGenerator
- .getEntityGreaterStartTime();
+ .getEntityGreaterStartTime(greaterStartTs);
te = new TimelineEntities();
te.addEntity(entityGreaterStartTime);
appName = "application_1000000000000000_2222";
@@ -183,24 +164,29 @@ public class TestHBaseStorageFlowRun {
.getBytes());
assertEquals(2, r1.size());
- Long starttime = (Long) GenericObjectMapper.read(values
+ long starttime = (Long) GenericObjectMapper.read(values
.get(FlowRunColumn.MIN_START_TIME.getColumnQualifierBytes()));
- Long expmin = entityMinStartTime.getCreatedTime();
- assertEquals(expmin, starttime);
+ assertEquals(minStartTs, starttime);
assertEquals(endTs, GenericObjectMapper.read(values
.get(FlowRunColumn.MAX_END_TIME.getColumnQualifierBytes())));
- }
- boolean isFlowRunRowKeyCorrect(byte[] rowKey, String cluster, String user,
- String flow, Long runid) {
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, -1);
- assertTrue(rowKeyComponents.length == 4);
- assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
- assertEquals(user, Bytes.toString(rowKeyComponents[1]));
- assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.invert(runid),
- Bytes.toLong(rowKeyComponents[3]));
- return true;
+ // use the timeline reader to verify data
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ // get the flow run entity
+ TimelineEntity entity =
+ hbr.getEntity(user, cluster, flow, runid, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null, null);
+ assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
+ FlowRunEntity flowRun = (FlowRunEntity)entity;
+ assertEquals(minStartTs, flowRun.getStartTime());
+ assertEquals(endTs, flowRun.getMaxEndTime());
+ } finally {
+ hbr.close();
+ }
}
/**
@@ -218,7 +204,7 @@ public class TestHBaseStorageFlowRun {
String user = "testWriteFlowRunMetricsOneFlow_user1";
String flow = "testing_flowRun_metrics_flow_name";
String flowVersion = "CF7022C10F1354";
- Long runid = 1002345678919L;
+ long runid = 1002345678919L;
TimelineEntities te = new TimelineEntities();
TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1();
@@ -244,6 +230,41 @@ public class TestHBaseStorageFlowRun {
// check flow run
checkFlowRunTable(cluster, user, flow, runid, c1);
+
+ // use the timeline reader to verify data
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ TimelineEntity entity =
+ hbr.getEntity(user, cluster, flow, runid, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null, null);
+ assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
+ Set<TimelineMetric> metrics = entity.getMetrics();
+ assertEquals(2, metrics.size());
+ for (TimelineMetric metric : metrics) {
+ String id = metric.getId();
+ Map<Long, Number> values = metric.getValues();
+ assertEquals(1, values.size());
+ Number value = null;
+ for (Number n : values.values()) {
+ value = n;
+ }
+ switch (id) {
+ case metric1:
+ assertEquals(141, value);
+ break;
+ case metric2:
+ assertEquals(57, value);
+ break;
+ default:
+ fail("unrecognized metric: " + id);
+ }
+ }
+ } finally {
+ hbr.close();
+ }
}
private void checkFlowRunTable(String cluster, String user, String flow,
[22/50] [abbrv] hadoop git commit: YARN-3908. Fixed bugs in
HBaseTimelineWriterImpl. Contributed by Vrushali C and Sangjin Lee.
Posted by sj...@apache.org.
YARN-3908. Fixed bugs in HBaseTimelineWriterImpl. Contributed by Vrushali C and Sangjin Lee.
(cherry picked from commit df0ec473a84871b0effd7ca6faac776210d7df09)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb4da07a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb4da07a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb4da07a
Branch: refs/heads/YARN-2928-rebase
Commit: eb4da07a87b054135678876f806dc2d19570d4cc
Parents: 0b2cff6
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Jul 27 15:50:28 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:39 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../records/timelineservice/TimelineEvent.java | 4 +-
.../storage/HBaseTimelineWriterImpl.java | 18 ++++++-
.../storage/common/ColumnHelper.java | 21 ++++----
.../storage/common/ColumnPrefix.java | 7 +--
.../storage/common/Separator.java | 7 +++
.../storage/entity/EntityColumnPrefix.java | 15 ++++--
.../storage/entity/EntityTable.java | 6 ++-
.../storage/TestHBaseTimelineWriterImpl.java | 56 ++++++++++++++++++--
9 files changed, 111 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4da07a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 564ca5a..e71c8f1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -115,6 +115,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3792. Test case failures in TestDistributedShell and some issue fixes
related to ATSV2 (Naganarasimha G R via sjlee)
+ YARN-3908. Fixed bugs in HBaseTimelineWriterImpl. (Vrushali C and Sangjin
+ Lee via zjshen)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4da07a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
index 1dbf7e5..a563658 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
@@ -33,6 +33,8 @@ import java.util.Map;
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class TimelineEvent implements Comparable<TimelineEvent> {
+ public static final long INVALID_TIMESTAMP = 0L;
+
private String id;
private HashMap<String, Object> info = new HashMap<>();
private long timestamp;
@@ -83,7 +85,7 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
}
public boolean isValid() {
- return (id != null && timestamp != 0L);
+ return (id != null && timestamp != INVALID_TIMESTAMP);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4da07a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 876ad6a..cd2e76e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -141,6 +141,13 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null,
te.getModifiedTime());
EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
+ Map<String, Object> info = te.getInfo();
+ if (info != null) {
+ for (Map.Entry<String, Object> entry : info.entrySet()) {
+ EntityColumnPrefix.INFO.store(rowKey, entityTable, entry.getKey(),
+ null, entry.getValue());
+ }
+ }
}
/**
@@ -186,6 +193,13 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
if (event != null) {
String eventId = event.getId();
if (eventId != null) {
+ long eventTimestamp = event.getTimestamp();
+ // if the timestamp is not set, use the current timestamp
+ if (eventTimestamp == TimelineEvent.INVALID_TIMESTAMP) {
+ LOG.warn("timestamp is not set for event " + eventId +
+ "! Using the current timestamp");
+ eventTimestamp = System.currentTimeMillis();
+ }
Map<String, Object> eventInfo = event.getInfo();
if (eventInfo != null) {
for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
@@ -198,8 +212,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// convert back to string to avoid additional API on store.
String compoundColumnQualifier =
Bytes.toString(compoundColumnQualifierBytes);
- EntityColumnPrefix.METRIC.store(rowKey, entityTable,
- compoundColumnQualifier, null, info.getValue());
+ EntityColumnPrefix.EVENT.store(rowKey, entityTable,
+ compoundColumnQualifier, eventTimestamp, info.getValue());
} // for info: eventInfo
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4da07a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
index 6a204dc..a902924 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -113,19 +113,22 @@ public class ColumnHelper<T> {
}
/**
- * @param result from which to reads timeseries data
+ * @param result from which to reads data with timestamps
* @param columnPrefixBytes optional prefix to limit columns. If null all
* columns are returned.
+ * @param <V> the type of the values. The values will be cast into that type.
* @return the cell values at each respective time in for form
* {idA={timestamp1->value1}, idA={timestamp2->value2},
* idB={timestamp3->value3}, idC={timestamp1->value4}}
* @throws IOException
*/
- public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
- Result result, byte[] columnPrefixBytes) throws IOException {
+ @SuppressWarnings("unchecked")
+ public <V> NavigableMap<String, NavigableMap<Long, V>>
+ readResultsWithTimestamps(Result result, byte[] columnPrefixBytes)
+ throws IOException {
- NavigableMap<String, NavigableMap<Long, Number>> results =
- new TreeMap<String, NavigableMap<Long, Number>>();
+ NavigableMap<String, NavigableMap<Long, V>> results =
+ new TreeMap<String, NavigableMap<Long, V>>();
if (result != null) {
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
@@ -157,13 +160,13 @@ public class ColumnHelper<T> {
// If this column has the prefix we want
if (columnName != null) {
- NavigableMap<Long, Number> cellResults =
- new TreeMap<Long, Number>();
+ NavigableMap<Long, V> cellResults =
+ new TreeMap<Long, V>();
NavigableMap<Long, byte[]> cells = entry.getValue();
if (cells != null) {
for (Entry<Long, byte[]> cell : cells.entrySet()) {
- Number value =
- (Number) GenericObjectMapper.read(cell.getValue());
+ V value =
+ (V) GenericObjectMapper.read(cell.getValue());
cellResults.put(cell.getKey(), value);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4da07a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
index 2eedea0..671c824 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
@@ -72,12 +72,13 @@ public interface ColumnPrefix<T> {
public Map<String, Object> readResults(Result result) throws IOException;
/**
- * @param result from which to reads timeseries data
+ * @param result from which to reads data with timestamps
+ * @param <V> the type of the values. The values will be cast into that type.
* @return the cell values at each respective time in for form
* {idA={timestamp1->value1}, idA={timestamp2->value2},
* idB={timestamp3->value3}, idC={timestamp1->value4}}
* @throws IOException
*/
- public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
- Result result) throws IOException;
+ public <V> NavigableMap<String, NavigableMap<Long, V>>
+ readResultsWithTimestamps(Result result) throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4da07a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
index ee57890..3319419 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
@@ -90,6 +90,13 @@ public enum Separator {
}
/**
+ * @return the original value of the separator
+ */
+ public String getValue() {
+ return value;
+ }
+
+ /**
* Used to make token safe to be used with this separator without collisions.
*
* @param token
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4da07a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index 4459868..8b7bc3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -45,6 +45,11 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
RELATES_TO(EntityColumnFamily.INFO, "r"),
/**
+ * To store TimelineEntity info values.
+ */
+ INFO(EntityColumnFamily.INFO, "i"),
+
+ /**
* Lifecycle events for an entity
*/
EVENT(EntityColumnFamily.INFO, "e"),
@@ -92,7 +97,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
/**
* @return the column name value
*/
- private String getColumnPrefix() {
+ public String getColumnPrefix() {
return columnPrefix;
}
@@ -150,11 +155,11 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
*
* @see
* org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readTimeseriesResults(org.apache.hadoop.hbase.client.Result)
+ * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
*/
- public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
- Result result) throws IOException {
- return column.readTimeseriesResults(result, columnPrefixBytes);
+ public <T> NavigableMap<String, NavigableMap<Long, T>>
+ readResultsWithTimestamps(Result result) throws IOException {
+ return column.readResultsWithTimestamps(result, columnPrefixBytes);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4da07a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 61f7c4c..2ae7d39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -54,7 +54,10 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineEnti
* | | modified_time: | | |
* | | 1392995081012 | metricId2: | |
* | | | metricValue1 | |
- * | | r!relatesToKey: | @timestamp2 | |
+ * | | i!infoKey: | @timestamp2 | |
+ * | | infoValue | | |
+ * | | | | |
+ * | | r!relatesToKey: | | |
* | | id3?id4?id5 | | |
* | | | | |
* | | s!isRelatedToKey | | |
@@ -62,6 +65,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineEnti
* | | | | |
* | | e!eventId?eventInfoKey: | | |
* | | eventInfoValue | | |
+ * | | @timestamp | | |
* | | | | |
* | | flowVersion: | | |
* | | versionValue | | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb4da07a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index 6abf240..31cb5d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -43,8 +43,10 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
@@ -84,6 +86,12 @@ public class TestHBaseTimelineWriterImpl {
entity.setCreatedTime(cTime);
entity.setModifiedTime(mTime);
+ // add the info map in Timeline Entity
+ Map<String, Object> infoMap = new HashMap<String, Object>();
+ infoMap.put("infoMapKey1", "infoMapValue1");
+ infoMap.put("infoMapKey2", 10);
+ entity.addInfo(infoMap);
+
// add the isRelatedToEntity info
String key = "task";
String value = "is_related_to_entity_id_here";
@@ -177,6 +185,14 @@ public class TestHBaseTimelineWriterImpl {
Long mTime1 = val.longValue();
assertEquals(mTime1, mTime);
+ Map<String, Object> infoColumns =
+ EntityColumnPrefix.INFO.readResults(result);
+ assertEquals(infoMap.size(), infoColumns.size());
+ for (String infoItem : infoMap.keySet()) {
+ assertEquals(infoMap.get(infoItem),
+ infoColumns.get(infoItem));
+ }
+
// Remember isRelatedTo is of type Map<String, Set<String>>
for (String isRelatedToKey : isRelatedTo.keySet()) {
Object isRelatedToValue =
@@ -219,7 +235,7 @@ public class TestHBaseTimelineWriterImpl {
}
NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- EntityColumnPrefix.METRIC.readTimeseriesResults(result);
+ EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
// We got metrics back
@@ -237,7 +253,7 @@ public class TestHBaseTimelineWriterImpl {
}
}
assertEquals(1, rowCount);
- assertEquals(15, colCount);
+ assertEquals(17, colCount);
} finally {
hbi.stop();
@@ -267,13 +283,18 @@ public class TestHBaseTimelineWriterImpl {
private void testAdditionalEntity() throws IOException {
TimelineEvent event = new TimelineEvent();
- event.setId("foo_event_id");
- event.setTimestamp(System.currentTimeMillis());
- event.addInfo("foo_event", "test");
+ String eventId = "foo_event_id";
+ event.setId(eventId);
+ Long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+ String expKey = "foo_event";
+ Object expVal = "test";
+ event.addInfo(expKey, expVal);
final TimelineEntity entity = new TimelineEntity();
entity.setId("attempt_1329348432655_0001_m_000008_18");
entity.setType("FOO_ATTEMPT");
+ entity.addEvent(event);
TimelineEntities entities = new TimelineEntities();
entities.addEntity(entity);
@@ -304,6 +325,31 @@ public class TestHBaseTimelineWriterImpl {
for (Result result : scanner) {
if (result != null && !result.isEmpty()) {
rowCount++;
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
+ entity));
+
+ // check the events
+ NavigableMap<String, NavigableMap<Long, Object>> eventsResult =
+ EntityColumnPrefix.EVENT.readResultsWithTimestamps(result);
+ // there should be only one event
+ assertEquals(1, eventsResult.size());
+ // key name for the event
+ String valueKey = eventId + Separator.VALUES.getValue() + expKey;
+ for (Map.Entry<String, NavigableMap<Long, Object>> e :
+ eventsResult.entrySet()) {
+ // the value key must match
+ assertEquals(valueKey, e.getKey());
+ NavigableMap<Long, Object> value = e.getValue();
+ // there should be only one timestamp and value
+ assertEquals(1, value.size());
+ for (Map.Entry<Long, Object> e2: value.entrySet()) {
+ assertEquals(expTs, e2.getKey());
+ assertEquals(expVal, e2.getValue());
+ }
+ }
}
}
assertEquals(1, rowCount);
[37/50] [abbrv] hadoop git commit: YARN-4058. Miscellaneous issues in
NodeManager project (Naganarasimha G R via sjlee)
Posted by sj...@apache.org.
YARN-4058. Miscellaneous issues in NodeManager project (Naganarasimha G R via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c51eceb3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c51eceb3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c51eceb3
Branch: refs/heads/YARN-2928-rebase
Commit: c51eceb3c79dd39991f57b5f2ea31106d633dfca
Parents: 5a34844
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Aug 24 17:36:31 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 16:52:32 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../containermanager/ContainerManagerImpl.java | 33 +++++++++++---------
.../metrics/TimelineServiceV2Publisher.java | 2 --
.../TestSystemMetricsPublisherForV2.java | 8 ++++-
4 files changed, 28 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c51eceb3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 75af072..43f615a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -144,6 +144,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-4064. build is broken at TestHBaseTimelineWriterImpl.java (sjlee)
+ YARN-4058. Miscellaneous issues in NodeManager project (Naganarasimha G R
+ via sjlee)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c51eceb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 12aab79..b010eee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -951,21 +951,24 @@ public class ContainerManagerImpl extends CompositeService implements
if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) {
flowRunId = Long.parseLong(flowRunIdStr);
}
- Application application = new ApplicationImpl(dispatcher, user,
- flowName, flowVersion, flowRunId, applicationID, credentials, context);
- if (null == context.getApplications().putIfAbsent(applicationID,
- application)) {
- LOG.info("Creating a new application reference for app " + applicationID);
- LogAggregationContext logAggregationContext =
- containerTokenIdentifier.getLogAggregationContext();
- Map<ApplicationAccessType, String> appAcls =
- container.getLaunchContext().getApplicationACLs();
- context.getNMStateStore().storeApplication(applicationID,
- buildAppProto(applicationID, user, credentials, appAcls,
- logAggregationContext));
- dispatcher.getEventHandler().handle(
- new ApplicationInitEvent(applicationID, appAcls,
- logAggregationContext));
+ if (!context.getApplications().containsKey(applicationID)) {
+ Application application =
+ new ApplicationImpl(dispatcher, user, flowName, flowVersion,
+ flowRunId, applicationID, credentials, context);
+ if (context.getApplications().putIfAbsent(applicationID,
+ application) == null) {
+ LOG.info("Creating a new application reference for app "
+ + applicationID);
+ LogAggregationContext logAggregationContext =
+ containerTokenIdentifier.getLogAggregationContext();
+ Map<ApplicationAccessType, String> appAcls =
+ container.getLaunchContext().getApplicationACLs();
+ context.getNMStateStore().storeApplication(applicationID,
+ buildAppProto(applicationID, user, credentials, appAcls,
+ logAggregationContext));
+ dispatcher.getEventHandler().handle(new ApplicationInitEvent(
+ applicationID, appAcls, logAggregationContext));
+ }
}
dispatcher.getEventHandler().handle(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c51eceb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 033e0c0..1a96e61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -39,8 +39,6 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identif
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c51eceb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 9830a80..ac20335 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -45,7 +46,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -332,6 +332,12 @@ public class TestSystemMetricsPublisherForV2 {
when(app.getRMAppMetrics()).thenReturn(
new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
when(app.getApplicationTags()).thenReturn(Collections.<String> emptySet());
+ ApplicationSubmissionContext appSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+ when(appSubmissionContext.getPriority())
+ .thenReturn(Priority.newInstance(0));
+ when(app.getApplicationSubmissionContext())
+ .thenReturn(appSubmissionContext);
return app;
}
[30/50] [abbrv] hadoop git commit: YARN-3049. [Storage
Implementation] Implement storage reader interface to fetch raw data from
HBase backend (Zhijie Shen via sjlee)
Posted by sj...@apache.org.
YARN-3049. [Storage Implementation] Implement storage reader interface to fetch raw data from HBase backend (Zhijie Shen via sjlee)
(cherry picked from commit 07433c2ad52df9e844dbd90020c277d3df844dcd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ecbdad0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ecbdad0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ecbdad0
Branch: refs/heads/YARN-2928-rebase
Commit: 2ecbdad0c7ed35358007f94afb57aee8211af995
Parents: 45ce22f
Author: Sangjin Lee <sj...@apache.org>
Authored: Fri Aug 7 10:00:22 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:41 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 6 +
.../records/timelineservice/TimelineEntity.java | 9 +-
.../storage/FileSystemTimelineReaderImpl.java | 164 +++----
.../storage/HBaseTimelineReaderImpl.java | 424 +++++++++++++++++++
.../storage/HBaseTimelineWriterImpl.java | 43 +-
.../storage/TimelineSchemaCreator.java | 12 +
.../storage/apptoflow/AppToFlowColumn.java | 126 ++++++
.../apptoflow/AppToFlowColumnFamily.java | 51 +++
.../storage/apptoflow/AppToFlowRowKey.java | 39 ++
.../storage/apptoflow/AppToFlowTable.java | 110 +++++
.../storage/apptoflow/package-info.java | 23 +
.../storage/common/BaseTable.java | 16 +
.../storage/common/ColumnPrefix.java | 2 +-
.../common/TimelineEntitySchemaConstants.java | 68 ---
.../common/TimelineHBaseSchemaConstants.java | 68 +++
.../storage/common/TimelineReaderUtils.java | 112 +++++
.../storage/entity/EntityColumn.java | 2 +-
.../storage/entity/EntityColumnFamily.java | 2 +-
.../storage/entity/EntityColumnPrefix.java | 2 +-
.../storage/entity/EntityRowKey.java | 36 +-
.../storage/entity/EntityTable.java | 8 +-
.../storage/TestHBaseTimelineWriterImpl.java | 82 +++-
23 files changed, 1198 insertions(+), 210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2cd44ad..922d54a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -82,6 +82,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3949. Ensure timely flush of timeline writes. (Sangjin Lee via
junping_du)
+ YARN-3049. [Storage Implementation] Implement storage reader interface to
+ fetch raw data from HBase backend (Zhijie Shen via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index d25d1d9..5583cd6 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -506,4 +506,10 @@
</Or>
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
+
+ <!-- Object cast is based on the event type -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.metrics.AbstractTimelineServicePublisher" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 9ef2d90..0701001 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -29,7 +29,9 @@ import javax.xml.bind.annotation.XmlRootElement;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.NavigableSet;
import java.util.Set;
+import java.util.TreeSet;
/**
* The basic timeline entity data structure for timeline service v2. Timeline
@@ -133,7 +135,8 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
private HashMap<String, Object> info = new HashMap<>();
private HashMap<String, String> configs = new HashMap<>();
private Set<TimelineMetric> metrics = new HashSet<>();
- private Set<TimelineEvent> events = new HashSet<>();
+ // events should be sorted by timestamp in descending order
+ private NavigableSet<TimelineEvent> events = new TreeSet<>();
private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>();
private HashMap<String, Set<String>> relatesToEntities = new HashMap<>();
private long createdTime;
@@ -334,7 +337,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
@XmlElement(name = "events")
- public Set<TimelineEvent> getEvents() {
+ public NavigableSet<TimelineEvent> getEvents() {
if (real == null) {
return events;
} else {
@@ -342,7 +345,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
}
- public void setEvents(Set<TimelineEvent> events) {
+ public void setEvents(NavigableSet<TimelineEvent> events) {
if (real == null) {
this.events = events;
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index f9f1d1d..45ddd1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.map.JsonMappingException;
@@ -119,59 +120,32 @@ public class FileSystemTimelineReaderImpl extends AbstractService
private static void fillFields(TimelineEntity finalEntity,
TimelineEntity real, EnumSet<Field> fields) {
if (fields.contains(Field.ALL)) {
- finalEntity.setConfigs(real.getConfigs());
- finalEntity.setMetrics(real.getMetrics());
- finalEntity.setInfo(real.getInfo());
- finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
- finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
- finalEntity.setEvents(real.getEvents());
- return;
+ fields = EnumSet.allOf(Field.class);
}
for (Field field : fields) {
switch(field) {
- case CONFIGS:
- finalEntity.setConfigs(real.getConfigs());
- break;
- case METRICS:
- finalEntity.setMetrics(real.getMetrics());
- break;
- case INFO:
- finalEntity.setInfo(real.getInfo());
- break;
- case IS_RELATED_TO:
- finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
- break;
- case RELATES_TO:
- finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
- break;
- case EVENTS:
- finalEntity.setEvents(real.getEvents());
- break;
- default:
- continue;
- }
- }
- }
-
- private static boolean matchFilter(Object infoValue, Object filterValue) {
- return infoValue.equals(filterValue);
- }
-
- private static boolean matchFilters(Map<String, ? extends Object> entityInfo,
- Map<String, ? extends Object> filters) {
- if (entityInfo == null || entityInfo.isEmpty()) {
- return false;
- }
- for (Map.Entry<String, ? extends Object> filter : filters.entrySet()) {
- Object infoValue = entityInfo.get(filter.getKey());
- if (infoValue == null) {
- return false;
- }
- if (!matchFilter(infoValue, filter.getValue())) {
- return false;
+ case CONFIGS:
+ finalEntity.setConfigs(real.getConfigs());
+ break;
+ case METRICS:
+ finalEntity.setMetrics(real.getMetrics());
+ break;
+ case INFO:
+ finalEntity.setInfo(real.getInfo());
+ break;
+ case IS_RELATED_TO:
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ break;
+ case RELATES_TO:
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ break;
+ case EVENTS:
+ finalEntity.setEvents(real.getEvents());
+ break;
+ default:
+ continue;
}
}
- return true;
}
private String getFlowRunPath(String userId, String clusterId, String flowId,
@@ -186,10 +160,10 @@ public class FileSystemTimelineReaderImpl extends AbstractService
String appFlowMappingFile = rootPath + "/" + ENTITIES_DIR + "/" +
clusterId + "/" + APP_FLOW_MAPPING_FILE;
try (BufferedReader reader =
- new BufferedReader(new InputStreamReader(
- new FileInputStream(
- appFlowMappingFile), Charset.forName("UTF-8")));
- CSVParser parser = new CSVParser(reader, csvFormat)) {
+ new BufferedReader(new InputStreamReader(
+ new FileInputStream(
+ appFlowMappingFile), Charset.forName("UTF-8")));
+ CSVParser parser = new CSVParser(reader, csvFormat)) {
for (CSVRecord record : parser.getRecords()) {
if (record.size() < 4) {
continue;
@@ -207,36 +181,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
throw new IOException("Unable to get flow info");
}
- private static boolean matchMetricFilters(Set<TimelineMetric> metrics,
- Set<String> metricFilters) {
- Set<String> tempMetrics = new HashSet<String>();
- for (TimelineMetric metric : metrics) {
- tempMetrics.add(metric.getId());
- }
-
- for (String metricFilter : metricFilters) {
- if (!tempMetrics.contains(metricFilter)) {
- return false;
- }
- }
- return true;
- }
-
- private static boolean matchEventFilters(Set<TimelineEvent> entityEvents,
- Set<String> eventFilters) {
- Set<String> tempEvents = new HashSet<String>();
- for (TimelineEvent event : entityEvents) {
- tempEvents.add(event.getId());
- }
-
- for (String eventFilter : eventFilters) {
- if (!tempEvents.contains(eventFilter)) {
- return false;
- }
- }
- return true;
- }
-
private static TimelineEntity createEntityToBeReturned(TimelineEntity entity,
EnumSet<Field> fieldsToRetrieve) {
TimelineEntity entityToBeReturned = new TimelineEntity();
@@ -254,23 +198,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
return (time >= timeBegin) && (time <= timeEnd);
}
- private static boolean matchRelations(
- Map<String, Set<String>> entityRelations,
- Map<String, Set<String>> relations) {
- for (Map.Entry<String, Set<String>> relation : relations.entrySet()) {
- Set<String> ids = entityRelations.get(relation.getKey());
- if (ids == null) {
- return false;
- }
- for (String id : relation.getValue()) {
- if (!ids.contains(id)) {
- return false;
- }
- }
- }
- return true;
- }
-
private static void mergeEntities(TimelineEntity entity1,
TimelineEntity entity2) {
// Ideally created time wont change except in the case of issue from client.
@@ -364,22 +291,22 @@ public class FileSystemTimelineReaderImpl extends AbstractService
// First sort the selected entities based on created/start time.
Map<Long, Set<TimelineEntity>> sortedEntities =
new TreeMap<>(
- new Comparator<Long>() {
- @Override
- public int compare(Long l1, Long l2) {
- return l2.compareTo(l1);
+ new Comparator<Long>() {
+ @Override
+ public int compare(Long l1, Long l2) {
+ return l2.compareTo(l1);
+ }
}
- }
);
for (File entityFile : dir.listFiles()) {
if (!entityFile.getName().contains(TIMELINE_SERVICE_STORAGE_EXTENSION)) {
continue;
}
try (BufferedReader reader =
- new BufferedReader(
- new InputStreamReader(
- new FileInputStream(
- entityFile), Charset.forName("UTF-8")))) {
+ new BufferedReader(
+ new InputStreamReader(
+ new FileInputStream(
+ entityFile), Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
if (!entity.getType().equals(entityType)) {
continue;
@@ -393,27 +320,32 @@ public class FileSystemTimelineReaderImpl extends AbstractService
continue;
}
if (relatesTo != null && !relatesTo.isEmpty() &&
- !matchRelations(entity.getRelatesToEntities(), relatesTo)) {
+ !TimelineReaderUtils
+ .matchRelations(entity.getRelatesToEntities(), relatesTo)) {
continue;
}
if (isRelatedTo != null && !isRelatedTo.isEmpty() &&
- !matchRelations(entity.getIsRelatedToEntities(), isRelatedTo)) {
+ !TimelineReaderUtils
+ .matchRelations(entity.getIsRelatedToEntities(), isRelatedTo)) {
continue;
}
if (infoFilters != null && !infoFilters.isEmpty() &&
- !matchFilters(entity.getInfo(), infoFilters)) {
+ !TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
continue;
}
if (configFilters != null && !configFilters.isEmpty() &&
- !matchFilters(entity.getConfigs(), configFilters)) {
+ !TimelineReaderUtils.matchFilters(
+ entity.getConfigs(), configFilters)) {
continue;
}
if (metricFilters != null && !metricFilters.isEmpty() &&
- !matchMetricFilters(entity.getMetrics(), metricFilters)) {
+ !TimelineReaderUtils.matchMetricFilters(
+ entity.getMetrics(), metricFilters)) {
continue;
}
if (eventFilters != null && !eventFilters.isEmpty() &&
- !matchEventFilters(entity.getEvents(), eventFilters)) {
+ !TimelineReaderUtils.matchEventFilters(
+ entity.getEvents(), eventFilters)) {
continue;
}
TimelineEntity entityToBeReturned =
@@ -461,8 +393,8 @@ public class FileSystemTimelineReaderImpl extends AbstractService
File entityFile =
new File(dir, entityId + TIMELINE_SERVICE_STORAGE_EXTENSION);
try (BufferedReader reader =
- new BufferedReader(new InputStreamReader(
- new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
+ new BufferedReader(new InputStreamReader(
+ new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
return createEntityToBeReturned(entity, fieldsToRetrieve);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
new file mode 100644
index 0000000..5258b9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -0,0 +1,424 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+public class HBaseTimelineReaderImpl
+ extends AbstractService implements TimelineReader {
+
+ private static final Log LOG = LogFactory
+ .getLog(HBaseTimelineReaderImpl.class);
+ private static final long DEFAULT_BEGIN_TIME = 0L;
+ private static final long DEFAULT_END_TIME = Long.MAX_VALUE;
+
+ private Configuration hbaseConf = null;
+ private Connection conn;
+ private EntityTable entityTable;
+ private AppToFlowTable appToFlowTable;
+
+ public HBaseTimelineReaderImpl() {
+ super(HBaseTimelineReaderImpl.class.getName());
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ hbaseConf = HBaseConfiguration.create(conf);
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ entityTable = new EntityTable();
+ appToFlowTable = new AppToFlowTable();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (conn != null) {
+ LOG.info("closing the hbase Connection");
+ conn.close();
+ }
+ super.serviceStop();
+ }
+
+ @Override
+ public TimelineEntity getEntity(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fieldsToRetrieve)
+ throws IOException {
+ validateParams(userId, clusterId, appId, entityType, entityId, true);
+ // In reality both should be null or neither should be null
+ if (flowId == null || flowRunId == null) {
+ FlowContext context = lookupFlowContext(clusterId, appId);
+ flowId = context.flowId;
+ flowRunId = context.flowRunId;
+ }
+ if (fieldsToRetrieve == null) {
+ fieldsToRetrieve = EnumSet.noneOf(Field.class);
+ }
+
+ byte[] rowKey = EntityRowKey.getRowKey(
+ clusterId, userId, flowId, flowRunId, appId, entityType, entityId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ return parseEntity(
+ entityTable.getResult(hbaseConf, conn, get), fieldsToRetrieve,
+ false, DEFAULT_BEGIN_TIME, DEFAULT_END_TIME, false, DEFAULT_BEGIN_TIME,
+ DEFAULT_END_TIME, null, null, null, null, null, null);
+ }
+
+ @Override
+ public Set<TimelineEntity> getEntities(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException {
+ validateParams(userId, clusterId, appId, entityType, null, false);
+ // In reality both should be null or neither should be null
+ if (flowId == null || flowRunId == null) {
+ FlowContext context = lookupFlowContext(clusterId, appId);
+ flowId = context.flowId;
+ flowRunId = context.flowRunId;
+ }
+ if (limit == null) {
+ limit = TimelineReader.DEFAULT_LIMIT;
+ }
+ if (createdTimeBegin == null) {
+ createdTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (createdTimeEnd == null) {
+ createdTimeEnd = DEFAULT_END_TIME;
+ }
+ if (modifiedTimeBegin == null) {
+ modifiedTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (modifiedTimeEnd == null) {
+ modifiedTimeEnd = DEFAULT_END_TIME;
+ }
+ if (fieldsToRetrieve == null) {
+ fieldsToRetrieve = EnumSet.noneOf(Field.class);
+ }
+
+ NavigableSet<TimelineEntity> entities = new TreeSet<>();
+ // Scan through part of the table to find the entities belong to one app and
+ // one type
+ Scan scan = new Scan();
+ scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
+ clusterId, userId, flowId, flowRunId, appId, entityType));
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ ResultScanner scanner = entityTable.getResultScanner(hbaseConf, conn, scan);
+ for (Result result : scanner) {
+ TimelineEntity entity = parseEntity(result, fieldsToRetrieve,
+ true, createdTimeBegin, createdTimeEnd,
+ true, modifiedTimeBegin, modifiedTimeEnd,
+ isRelatedTo, relatesTo, infoFilters, configFilters, eventFilters,
+ metricFilters);
+ if (entity == null) {
+ continue;
+ }
+ if (entities.size() > limit) {
+ entities.pollLast();
+ }
+ entities.add(entity);
+ }
+ return entities;
+ }
+
+ private FlowContext lookupFlowContext(String clusterId, String appId)
+ throws IOException {
+ byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
+ Get get = new Get(rowKey);
+ Result result = appToFlowTable.getResult(hbaseConf, conn, get);
+ if (result != null && !result.isEmpty()) {
+ return new FlowContext(
+ AppToFlowColumn.FLOW_ID.readResult(result).toString(),
+ ((Number) AppToFlowColumn.FLOW_RUN_ID.readResult(result)).longValue());
+ } else {
+ throw new IOException(
+ "Unable to find the context flow ID and flow run ID for clusterId=" +
+ clusterId + ", appId=" + appId);
+ }
+ }
+
+ private static class FlowContext {
+ private String flowId;
+ private Long flowRunId;
+ public FlowContext(String flowId, Long flowRunId) {
+ this.flowId = flowId;
+ this.flowRunId = flowRunId;
+ }
+ }
+
+ private static void validateParams(String userId, String clusterId,
+ String appId, String entityType, String entityId, boolean checkEntityId) {
+ Preconditions.checkNotNull(userId, "userId shouldn't be null");
+ Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ Preconditions.checkNotNull(appId, "appId shouldn't be null");
+ Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
+ if (checkEntityId) {
+ Preconditions.checkNotNull(entityId, "entityId shouldn't be null");
+ }
+ }
+
+ private static TimelineEntity parseEntity(
+ Result result, EnumSet<Field> fieldsToRetrieve,
+ boolean checkCreatedTime, long createdTimeBegin, long createdTimeEnd,
+ boolean checkModifiedTime, long modifiedTimeBegin, long modifiedTimeEnd,
+ Map<String, Set<String>> isRelatedTo, Map<String, Set<String>> relatesTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> eventFilters, Set<String> metricFilters)
+ throws IOException {
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ TimelineEntity entity = new TimelineEntity();
+ entity.setType(EntityColumn.TYPE.readResult(result).toString());
+ entity.setId(EntityColumn.ID.readResult(result).toString());
+
+ // fetch created time
+ entity.setCreatedTime(
+ ((Number) EntityColumn.CREATED_TIME.readResult(result)).longValue());
+ if (checkCreatedTime && (entity.getCreatedTime() < createdTimeBegin ||
+ entity.getCreatedTime() > createdTimeEnd)) {
+ return null;
+ }
+
+ // fetch modified time
+ entity.setCreatedTime(
+ ((Number) EntityColumn.MODIFIED_TIME.readResult(result)).longValue());
+ if (checkModifiedTime && (entity.getModifiedTime() < modifiedTimeBegin ||
+ entity.getModifiedTime() > modifiedTimeEnd)) {
+ return null;
+ }
+
+ // fetch is related to entities
+ boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
+ readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO);
+ if (checkIsRelatedTo && !TimelineReaderUtils.matchRelations(
+ entity.getIsRelatedToEntities(), isRelatedTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
+ entity.getIsRelatedToEntities().clear();
+ }
+ }
+
+ // fetch relates to entities
+ boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
+ readRelationship(entity, result, EntityColumnPrefix.RELATES_TO);
+ if (checkRelatesTo && !TimelineReaderUtils.matchRelations(
+ entity.getRelatesToEntities(), relatesTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.RELATES_TO)) {
+ entity.getRelatesToEntities().clear();
+ }
+ }
+
+ // fetch info
+ boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.INFO);
+ if (checkInfo &&
+ !TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.INFO)) {
+ entity.getInfo().clear();
+ }
+ }
+
+ // fetch configs
+ boolean checkConfigs = configFilters != null && configFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG);
+ if (checkConfigs && !TimelineReaderUtils.matchFilters(
+ entity.getConfigs(), configFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.CONFIGS)) {
+ entity.getConfigs().clear();
+ }
+ }
+
+ // fetch events
+ boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
+ readEvents(entity, result);
+ if (checkEvents && !TimelineReaderUtils.matchEventFilters(
+ entity.getEvents(), eventFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.EVENTS)) {
+ entity.getEvents().clear();
+ }
+ }
+
+ // fetch metrics
+ boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
+ readMetrics(entity, result);
+ if (checkMetrics && !TimelineReaderUtils.matchMetricFilters(
+ entity.getMetrics(), metricFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.METRICS)) {
+ entity.getMetrics().clear();
+ }
+ }
+ return entity;
+ }
+
+ private static void readRelationship(
+ TimelineEntity entity, Result result, EntityColumnPrefix prefix)
+ throws IOException {
+ // isRelatedTo and relatesTo are of type Map<String, Set<String>>
+ Map<String, Object> columns = prefix.readResults(result);
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ for (String id : Separator.VALUES.splitEncoded(
+ column.getValue().toString())) {
+ if (prefix.equals(EntityColumnPrefix.IS_RELATED_TO)) {
+ entity.addIsRelatedToEntity(column.getKey(), id);
+ } else {
+ entity.addRelatesToEntity(column.getKey(), id);
+ }
+ }
+ }
+ }
+
+ private static void readKeyValuePairs(
+ TimelineEntity entity, Result result, EntityColumnPrefix prefix)
+ throws IOException {
+ // info and configuration are of type Map<String, Object or String>
+ Map<String, Object> columns = prefix.readResults(result);
+ if (prefix.equals(EntityColumnPrefix.CONFIG)) {
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ entity.addConfig(column.getKey(), column.getKey().toString());
+ }
+ } else {
+ entity.addInfo(columns);
+ }
+ }
+
+ private static void readEvents(TimelineEntity entity, Result result)
+ throws IOException {
+ Map<String, TimelineEvent> eventsMap = new HashMap<>();
+ Map<String, Object> eventsResult =
+ EntityColumnPrefix.EVENT.readResults(result);
+ for (Map.Entry<String,Object> eventResult : eventsResult.entrySet()) {
+ Collection<String> tokens =
+ Separator.VALUES.splitEncoded(eventResult.getKey());
+ if (tokens.size() != 2 && tokens.size() != 3) {
+ throw new IOException(
+ "Invalid event column name: " + eventResult.getKey());
+ }
+ Iterator<String> idItr = tokens.iterator();
+ String id = idItr.next();
+ String tsStr = idItr.next();
+ // TODO: timestamp is not correct via ser/des through UTF-8 string
+ Long ts =
+ TimelineWriterUtils.invert(Bytes.toLong(tsStr.getBytes(
+ StandardCharsets.UTF_8)));
+ String key = Separator.VALUES.joinEncoded(id, ts.toString());
+ TimelineEvent event = eventsMap.get(key);
+ if (event == null) {
+ event = new TimelineEvent();
+ event.setId(id);
+ event.setTimestamp(ts);
+ eventsMap.put(key, event);
+ }
+ if (tokens.size() == 3) {
+ String infoKey = idItr.next();
+ event.addInfo(infoKey, eventResult.getValue());
+ }
+ }
+ Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
+ entity.addEvents(eventsSet);
+ }
+
+ private static void readMetrics(TimelineEntity entity, Result result)
+ throws IOException {
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
+ for (Map.Entry<String, NavigableMap<Long, Number>> metricResult:
+ metricsResult.entrySet()) {
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId(metricResult.getKey());
+ // Simply assume that if the value set contains more than 1 elements, the
+ // metric is a TIME_SERIES metric, otherwise, it's a SINGLE_VALUE metric
+ metric.setType(metricResult.getValue().size() > 1 ?
+ TimelineMetric.Type.TIME_SERIES : TimelineMetric.Type.SINGLE_VALUE);
+ metric.addValues(metricResult.getValue());
+ entity.addMetric(metric);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 3173e87..5290415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -33,9 +33,14 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
@@ -55,6 +60,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
private Connection conn;
private TypedBufferedMutator<EntityTable> entityTable;
+ private TypedBufferedMutator<AppToFlowTable> appToFlowTable;
private static final Log LOG = LogFactory
.getLog(HBaseTimelineWriterImpl.class);
@@ -77,6 +83,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
Configuration hbaseConf = HBaseConfiguration.create(conf);
conn = ConnectionFactory.createConnection(hbaseConf);
entityTable = new EntityTable().getTableMutator(hbaseConf, conn);
+ appToFlowTable = new AppToFlowTable().getTableMutator(hbaseConf, conn);
}
/**
@@ -97,7 +104,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
byte[] rowKey =
EntityRowKey.getRowKey(clusterId, userId, flowName, flowRunId, appId,
- te);
+ te.getType(), te.getId());
storeInfo(rowKey, te, flowVersion);
storeEvents(rowKey, te.getEvents());
@@ -107,11 +114,37 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
EntityColumnPrefix.IS_RELATED_TO);
storeRelations(rowKey, te.getRelatesToEntities(),
EntityColumnPrefix.RELATES_TO);
- }
+ if (isApplicationCreated(te)) {
+ onApplicationCreated(
+ clusterId, userId, flowName, flowVersion, flowRunId, appId, te);
+ }
+ }
return putStatus;
}
+ private static boolean isApplicationCreated(TimelineEntity te) {
+ if (te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString())) {
+ boolean isAppCreated = false;
+ for (TimelineEvent event : te.getEvents()) {
+ if (event.getId().equals(
+ ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private void onApplicationCreated(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRunId, String appId,
+ TimelineEntity te) throws IOException {
+ byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
+ AppToFlowColumn.FLOW_ID.store(rowKey, appToFlowTable, null, flowName);
+ AppToFlowColumn.FLOW_RUN_ID.store(
+ rowKey, appToFlowTable, null, flowRunId);
+ }
+
/**
* Stores the Relations from the {@linkplain TimelineEntity} object
*/
@@ -245,6 +278,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
public void flush() throws IOException {
// flush all buffered mutators
entityTable.flush();
+ appToFlowTable.flush();
}
/**
@@ -258,6 +292,11 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// The close API performs flushing and releases any resources held
entityTable.close();
}
+ if (appToFlowTable != null) {
+ LOG.info("closing app_flow table");
+ // The close API performs flushing and releases any resources held
+ appToFlowTable.close();
+ }
if (conn != null) {
LOG.info("closing the hbase Connection");
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index a5cc2ab..2c3897d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
/**
@@ -70,6 +71,11 @@ public class TimelineSchemaCreator {
int metricsTTL = Integer.parseInt(entityTableTTLMetrics);
new EntityTable().setMetricsTTL(metricsTTL, hbaseConf);
}
+ // Grab the appToflowTableName argument
+ String appToflowTableName = commandLine.getOptionValue("a2f");
+ if (StringUtils.isNotBlank(appToflowTableName)) {
+ hbaseConf.set(AppToFlowTable.TABLE_NAME_CONF_NAME, appToflowTableName);
+ }
createAllTables(hbaseConf);
}
@@ -95,6 +101,11 @@ public class TimelineSchemaCreator {
o.setRequired(false);
options.addOption(o);
+ o = new Option("a2f", "appToflowTableName", true, "app to flow table name");
+ o.setArgName("appToflowTableName");
+ o.setRequired(false);
+ options.addOption(o);
+
CommandLineParser parser = new PosixParser();
CommandLine commandLine = null;
try {
@@ -120,6 +131,7 @@ public class TimelineSchemaCreator {
throw new IOException("Cannot create table since admin is null");
}
new EntityTable().createTable(admin, hbaseConf);
+ new AppToFlowTable().createTable(admin, hbaseConf);
} finally {
if (conn != null) {
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
new file mode 100644
index 0000000..423037a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+import java.io.IOException;
+
+/**
+ * Identifies fully qualified columns for the {@link AppToFlowTable}.
+ */
+public enum AppToFlowColumn implements Column<AppToFlowTable> {
+
+ /**
+ * The flow ID
+ */
+ FLOW_ID(AppToFlowColumnFamily.MAPPING, "flow_id"),
+
+ /**
+ * The flow run ID
+ */
+ FLOW_RUN_ID(AppToFlowColumnFamily.MAPPING, "flow_run_id");
+
+ private final ColumnHelper<AppToFlowTable> column;
+ private final ColumnFamily<AppToFlowTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+
+ AppToFlowColumn(ColumnFamily<AppToFlowTable> columnFamily,
+ String columnQualifier) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.column = new ColumnHelper<AppToFlowTable>(columnFamily);
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<AppToFlowTable> tableMutator, Long timestamp,
+ Object inputValue) throws IOException {
+ column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
+ inputValue);
+ }
+
+ public Object readResult(Result result) throws IOException {
+ return column.readResult(result, columnQualifierBytes);
+ }
+
+ /**
+ * Retrieve an {@link AppToFlowColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(x) == columnFor(y)} if
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnQualifier Name of the column to retrieve
+ * @return the corresponding {@link AppToFlowColumn} or null
+ */
+ public static final AppToFlowColumn columnFor(String columnQualifier) {
+
+ // Match column based on value, assume column family matches.
+ for (AppToFlowColumn ec : AppToFlowColumn.values()) {
+ // Find a match based only on name.
+ if (ec.getColumnQualifier().equals(columnQualifier)) {
+ return ec;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link AppToFlowColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)}
+ * if and only if {@code a.equals(b) & x.equals(y)} or
+ * {@code (x == y == null)}
+ *
+ * @param columnFamily The columnFamily for which to retrieve the column.
+ * @param name Name of the column to retrieve
+ * @return the corresponding {@link AppToFlowColumn} or null if both arguments
+ * don't match.
+ */
+ public static final AppToFlowColumn columnFor(
+ AppToFlowColumnFamily columnFamily, String name) {
+
+ for (AppToFlowColumn ec : AppToFlowColumn.values()) {
+ // Find a match based column family and on name.
+ if (ec.columnFamily.equals(columnFamily)
+ && ec.getColumnQualifier().equals(name)) {
+ return ec;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
new file mode 100644
index 0000000..e74235f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the app_flow table column families.
+ */
+public enum AppToFlowColumnFamily implements ColumnFamily<AppToFlowTable> {
+ /**
+ * Mapping column family houses known columns such as flowId and flowRunId
+ */
+ MAPPING("m");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ AppToFlowColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
new file mode 100644
index 0000000..ad4fec6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the app_flow table.
+ */
+public class AppToFlowRowKey {
+ /**
+ * Constructs a row key prefix for the app_flow table as follows:
+ * {@code clusterId!AppId}
+ *
+ * @param clusterId
+ * @param appId
+ * @return byte array with the row key
+ */
+ public static byte[] getRowKey(String clusterId, String appId) {
+ return Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId, appId));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
new file mode 100644
index 0000000..2467856
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
+
+import java.io.IOException;
+
+/**
+ * The app_flow table as column families mapping. Mapping stores
+ * appId to flowId and flowRunId mapping information
+ *
+ * Example app_flow table record:
+ *
+ * <pre>
+ * |--------------------------------------|
+ * | Row | Column Family |
+ * | key | info |
+ * |--------------------------------------|
+ * | clusterId! | flowId: |
+ * | AppId | foo@daily_hive_report |
+ * | | |
+ * | | flowRunId: |
+ * | | 1452828720457 |
+ * | | |
+ * | | |
+ * | | |
+ * |--------------------------------------|
+ * </pre>
+ */
+public class AppToFlowTable extends BaseTable<AppToFlowTable> {
+ /** app_flow prefix */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "app-flow";
+
+ /** config param name that specifies the app_flow table name */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /** default value for app_flow table name */
+ private static final String DEFAULT_TABLE_NAME = "timelineservice.app_flow";
+
+ private static final Log LOG = LogFactory.getLog(AppToFlowTable.class);
+
+ public AppToFlowTable() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
+ * (org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor appToFlowTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor mappCF =
+ new HColumnDescriptor(AppToFlowColumnFamily.MAPPING.getBytes());
+ mappCF.setBloomFilterType(BloomType.ROWCOL);
+ appToFlowTableDescp.addFamily(mappCF);
+
+ appToFlowTableDescp
+ .setRegionSplitPolicyClassName(
+ "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ appToFlowTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(appToFlowTableDescp,
+ TimelineHBaseSchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
new file mode 100644
index 0000000..df7ffc1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
index e8d8b5c..abba79a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.BufferedMutator;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
@@ -94,6 +96,20 @@ public abstract class BaseTable<T> {
}
/**
+ *
+ * @param hbaseConf used to read settings that override defaults
+ * @param conn used to create table from
+ * @param get that specifies what single row you want to get from this table
+ * @return result of get operation
+ * @throws IOException
+ */
+ public Result getResult(Configuration hbaseConf, Connection conn, Get get)
+ throws IOException {
+ Table table = conn.getTable(getTableName(hbaseConf));
+ return table.get(get);
+ }
+
+ /**
* Get the table name for this table.
*
* @param hbaseConf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
index 671c824..509ff49 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
@@ -64,7 +64,7 @@ public interface ColumnPrefix<T> {
public Object readResult(Result result, String qualifier) throws IOException;
/**
- * @param resultfrom which to read columns
+ * @param result from which to read columns
* @return the latest values of columns in the column family with this prefix
* (or all of them if the prefix value is null).
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
deleted file mode 100644
index 5518a27..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * contains the constants used in the context of schema accesses for
- * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
- * information
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class TimelineEntitySchemaConstants {
-
- /**
- * Used to create a pre-split for tables starting with a username in the
- * prefix. TODO: this may have to become a config variable (string with
- * separators) so that different installations can presplit based on their own
- * commonly occurring names.
- */
- private final static byte[][] USERNAME_SPLITS = { Bytes.toBytes("a"),
- Bytes.toBytes("ad"), Bytes.toBytes("an"), Bytes.toBytes("b"),
- Bytes.toBytes("ca"), Bytes.toBytes("cl"), Bytes.toBytes("d"),
- Bytes.toBytes("e"), Bytes.toBytes("f"), Bytes.toBytes("g"),
- Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j"),
- Bytes.toBytes("k"), Bytes.toBytes("l"), Bytes.toBytes("m"),
- Bytes.toBytes("n"), Bytes.toBytes("o"), Bytes.toBytes("q"),
- Bytes.toBytes("r"), Bytes.toBytes("s"), Bytes.toBytes("se"),
- Bytes.toBytes("t"), Bytes.toBytes("u"), Bytes.toBytes("v"),
- Bytes.toBytes("w"), Bytes.toBytes("x"), Bytes.toBytes("y"),
- Bytes.toBytes("z") };
-
- /**
- * The length at which keys auto-split
- */
- public static final String USERNAME_SPLIT_KEY_PREFIX_LENGTH = "4";
-
- /**
- * @return splits for splits where a user is a prefix.
- */
- public final static byte[][] getUsernameSplits() {
- byte[][] kloon = USERNAME_SPLITS.clone();
- // Deep copy.
- for (int row = 0; row < USERNAME_SPLITS.length; row++) {
- kloon[row] = Bytes.copy(USERNAME_SPLITS[row]);
- }
- return kloon;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
new file mode 100644
index 0000000..bbf498a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * contains the constants used in the context of schema accesses for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * information
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class TimelineHBaseSchemaConstants {
+
+ /**
+ * Used to create a pre-split for tables starting with a username in the
+ * prefix. TODO: this may have to become a config variable (string with
+ * separators) so that different installations can presplit based on their own
+ * commonly occurring names.
+ */
+ private final static byte[][] USERNAME_SPLITS = { Bytes.toBytes("a"),
+ Bytes.toBytes("ad"), Bytes.toBytes("an"), Bytes.toBytes("b"),
+ Bytes.toBytes("ca"), Bytes.toBytes("cl"), Bytes.toBytes("d"),
+ Bytes.toBytes("e"), Bytes.toBytes("f"), Bytes.toBytes("g"),
+ Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j"),
+ Bytes.toBytes("k"), Bytes.toBytes("l"), Bytes.toBytes("m"),
+ Bytes.toBytes("n"), Bytes.toBytes("o"), Bytes.toBytes("q"),
+ Bytes.toBytes("r"), Bytes.toBytes("s"), Bytes.toBytes("se"),
+ Bytes.toBytes("t"), Bytes.toBytes("u"), Bytes.toBytes("v"),
+ Bytes.toBytes("w"), Bytes.toBytes("x"), Bytes.toBytes("y"),
+ Bytes.toBytes("z") };
+
+ /**
+ * The length at which keys auto-split
+ */
+ public static final String USERNAME_SPLIT_KEY_PREFIX_LENGTH = "4";
+
+ /**
+ * @return splits for splits where a user is a prefix.
+ */
+ public final static byte[][] getUsernameSplits() {
+ byte[][] kloon = USERNAME_SPLITS.clone();
+ // Deep copy.
+ for (int row = 0; row < USERNAME_SPLITS.length; row++) {
+ kloon[row] = Bytes.copy(USERNAME_SPLITS[row]);
+ }
+ return kloon;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java
new file mode 100644
index 0000000..91d7ba4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+
+public class TimelineReaderUtils {
+ /**
+ *
+ * @param entityRelations the relations of an entity
+ * @param relationFilters the relations for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchRelations(
+ Map<String, Set<String>> entityRelations,
+ Map<String, Set<String>> relationFilters) {
+ for (Map.Entry<String, Set<String>> relation : relationFilters.entrySet()) {
+ Set<String> ids = entityRelations.get(relation.getKey());
+ if (ids == null) {
+ return false;
+ }
+ for (String id : relation.getValue()) {
+ if (!ids.contains(id)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ *
+ * @param map the map of key/value pairs in an entity
+ * @param filters the map of key/value pairs for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchFilters(Map<String, ? extends Object> map,
+ Map<String, ? extends Object> filters) {
+ for (Map.Entry<String, ? extends Object> filter : filters.entrySet()) {
+ Object value = map.get(filter.getKey());
+ if (value == null) {
+ return false;
+ }
+ if (!value.equals(filter.getValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ *
+ * @param entityEvents the set of event objects in an entity
+ * @param eventFilters the set of event Ids for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchEventFilters(Set<TimelineEvent> entityEvents,
+ Set<String> eventFilters) {
+ Set<String> eventIds = new HashSet<String>();
+ for (TimelineEvent event : entityEvents) {
+ eventIds.add(event.getId());
+ }
+ for (String eventFilter : eventFilters) {
+ if (!eventIds.contains(eventFilter)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ *
+ * @param metrics the set of metric objects in an entity
+ * @param metricFilters the set of metric Ids for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchMetricFilters(Set<TimelineMetric> metrics,
+ Set<String> metricFilters) {
+ Set<String> metricIds = new HashSet<String>();
+ for (TimelineMetric metric : metrics) {
+ metricIds.add(metric.getId());
+ }
+
+ for (String metricFilter : metricFilters) {
+ if (!metricIds.contains(metricFilter)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
index 90da966..26e7748 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
@@ -62,7 +62,7 @@ public enum EntityColumn implements Column<EntityTable> {
private final String columnQualifier;
private final byte[] columnQualifierBytes;
- private EntityColumn(ColumnFamily<EntityTable> columnFamily,
+ EntityColumn(ColumnFamily<EntityTable> columnFamily,
String columnQualifier) {
this.columnFamily = columnFamily;
this.columnQualifier = columnQualifier;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
index 8a95d12..7c63727 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
@@ -53,7 +53,7 @@ public enum EntityColumnFamily implements ColumnFamily<EntityTable> {
* @param value create a column family with this name. Must be lower case and
* without spaces.
*/
- private EntityColumnFamily(String value) {
+ EntityColumnFamily(String value) {
// column families should be lower case and not contain any spaces.
this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index 8b7bc3e..58272ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -80,7 +80,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
* @param columnFamily that this column is stored in.
* @param columnPrefix for this column.
*/
- private EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
+ EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
String columnPrefix) {
column = new ColumnHelper<EntityTable>(columnFamily);
this.columnFamily = columnFamily;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
index 3e17ad0..9a72be0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -55,17 +55,45 @@ public class EntityRowKey {
/**
* Constructs a row key prefix for the entity table as follows:
- * {@code userName!clusterId!flowId!flowRunId!AppId}
+ * {@code userName!clusterId!flowId!flowRunId!AppId!entityType!}
*
* @param clusterId
* @param userId
* @param flowId
* @param flowRunId
* @param appId
+ * @param entityType
* @return byte array with the row key prefix
*/
+ public static byte[] getRowKeyPrefix(String clusterId, String userId,
+ String flowId, Long flowRunId, String appId, String entityType) {
+ byte[] first =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
+ flowId));
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
+ byte[] third =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, entityType, ""));
+ return Separator.QUALIFIERS.join(first, second, third);
+ }
+
+ /**
+ * Constructs a row key for the entity table as follows:
+ * {@code userName!clusterId!flowId!flowRunId!AppId!entityType!entityId}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @param appId
+ * @param entityType
+ * @param entityId
+ * @return byte array with the row key
+ */
public static byte[] getRowKey(String clusterId, String userId,
- String flowId, Long flowRunId, String appId, TimelineEntity te) {
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId) {
byte[] first =
Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
flowId));
@@ -73,8 +101,8 @@ public class EntityRowKey {
// time.
byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
byte[] third =
- Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, te.getType(),
- te.getId()));
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, entityType,
+ entityId));
return Separator.QUALIFIERS.join(first, second, third);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 2ae7d39..f657a14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineEntitySchemaConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
/**
* The entity table as column families info, config and metrics. Info stores
@@ -75,7 +75,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineEnti
public class EntityTable extends BaseTable<EntityTable> {
/** entity prefix */
private static final String PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".entity";
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "entity";
/** config param name that specifies the entity table name */
public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
@@ -146,9 +146,9 @@ public class EntityTable extends BaseTable<EntityTable> {
entityTableDescp
.setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
- TimelineEntitySchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
admin.createTable(entityTableDescp,
- TimelineEntitySchemaConstants.getUsernameSplits());
+ TimelineHBaseSchemaConstants.getUsernameSplits());
LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ admin.tableExists(table));
}
[49/50] [abbrv] hadoop git commit: YARN-4178. [storage
implementation] app id as string in row keys can cause incorrect ordering
(Varun Saxena via sjlee)
Posted by sj...@apache.org.
YARN-4178. [storage implementation] app id as string in row keys can cause incorrect ordering (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fec6f63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fec6f63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fec6f63
Branch: refs/heads/YARN-2928-rebase
Commit: 6fec6f63b56c47922aad03923225a8516d94b7c0
Parents: cde783f
Author: Sangjin Lee <sj...@apache.org>
Authored: Tue Oct 6 16:06:28 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 17:05:06 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../storage/ApplicationEntityReader.java | 14 +-
.../storage/FileSystemTimelineReaderImpl.java | 14 +-
.../storage/GenericEntityReader.java | 17 +-
.../storage/HBaseTimelineWriterImpl.java | 20 +-
.../storage/application/ApplicationRowKey.java | 13 +-
.../storage/apptoflow/AppToFlowRowKey.java | 7 +-
.../storage/common/Separator.java | 4 +-
.../storage/common/TimelineReaderUtils.java | 112 -----
.../storage/common/TimelineStorageUtils.java | 475 +++++++++++++++++++
.../storage/common/TimelineWriterUtils.java | 328 -------------
.../storage/entity/EntityRowKey.java | 32 +-
.../storage/flow/FlowActivityColumnPrefix.java | 6 +-
.../storage/flow/FlowActivityRowKey.java | 9 +-
.../storage/flow/FlowRunColumn.java | 4 +-
.../storage/flow/FlowRunColumnPrefix.java | 6 +-
.../storage/flow/FlowRunCoprocessor.java | 4 +-
.../storage/flow/FlowRunRowKey.java | 6 +-
.../storage/flow/FlowScanner.java | 6 +-
.../storage/TestHBaseTimelineStorage.java | 13 +-
.../common/TestTimelineStorageUtils.java | 56 +++
.../storage/common/TestTimelineWriterUtils.java | 29 --
.../flow/TestHBaseStorageFlowActivity.java | 10 +-
23 files changed, 629 insertions(+), 559 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 969fcec..b24a178 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -168,6 +168,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-4058. Miscellaneous issues in NodeManager project (Naganarasimha G R
via sjlee)
+ YARN-4178. [storage implementation] app id as string in row keys can cause
+ incorrect ordering (Varun Saxena via sjlee)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
index 61954e1..6d1a2ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.application.Applica
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import com.google.common.base.Preconditions;
@@ -182,7 +182,7 @@ class ApplicationEntityReader extends GenericEntityReader {
fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
true);
- if (checkIsRelatedTo && !TimelineReaderUtils.matchRelations(
+ if (checkIsRelatedTo && !TimelineStorageUtils.matchRelations(
entity.getIsRelatedToEntities(), isRelatedTo)) {
return null;
}
@@ -198,7 +198,7 @@ class ApplicationEntityReader extends GenericEntityReader {
fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
false);
- if (checkRelatesTo && !TimelineReaderUtils.matchRelations(
+ if (checkRelatesTo && !TimelineStorageUtils.matchRelations(
entity.getRelatesToEntities(), relatesTo)) {
return null;
}
@@ -214,7 +214,7 @@ class ApplicationEntityReader extends GenericEntityReader {
fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
if (checkInfo &&
- !TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -228,7 +228,7 @@ class ApplicationEntityReader extends GenericEntityReader {
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
- if (checkConfigs && !TimelineReaderUtils.matchFilters(
+ if (checkConfigs && !TimelineStorageUtils.matchFilters(
entity.getConfigs(), configFilters)) {
return null;
}
@@ -243,7 +243,7 @@ class ApplicationEntityReader extends GenericEntityReader {
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
readEvents(entity, result, true);
- if (checkEvents && !TimelineReaderUtils.matchEventFilters(
+ if (checkEvents && !TimelineStorageUtils.matchEventFilters(
entity.getEvents(), eventFilters)) {
return null;
}
@@ -258,7 +258,7 @@ class ApplicationEntityReader extends GenericEntityReader {
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
readMetrics(entity, result, ApplicationColumnPrefix.METRIC);
- if (checkMetrics && !TimelineReaderUtils.matchMetricFilters(
+ if (checkMetrics && !TimelineStorageUtils.matchMetricFilters(
entity.getMetrics(), metricFilters)) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index 626c770..30d1d00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.map.JsonMappingException;
@@ -321,31 +321,31 @@ public class FileSystemTimelineReaderImpl extends AbstractService
continue;
}
if (relatesTo != null && !relatesTo.isEmpty() &&
- !TimelineReaderUtils
+ !TimelineStorageUtils
.matchRelations(entity.getRelatesToEntities(), relatesTo)) {
continue;
}
if (isRelatedTo != null && !isRelatedTo.isEmpty() &&
- !TimelineReaderUtils
+ !TimelineStorageUtils
.matchRelations(entity.getIsRelatedToEntities(), isRelatedTo)) {
continue;
}
if (infoFilters != null && !infoFilters.isEmpty() &&
- !TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) {
continue;
}
if (configFilters != null && !configFilters.isEmpty() &&
- !TimelineReaderUtils.matchFilters(
+ !TimelineStorageUtils.matchFilters(
entity.getConfigs(), configFilters)) {
continue;
}
if (metricFilters != null && !metricFilters.isEmpty() &&
- !TimelineReaderUtils.matchMetricFilters(
+ !TimelineStorageUtils.matchMetricFilters(
entity.getMetrics(), metricFilters)) {
continue;
}
if (eventFilters != null && !eventFilters.isEmpty() &&
- !TimelineReaderUtils.matchEventFilters(
+ !TimelineStorageUtils.matchEventFilters(
entity.getEvents(), eventFilters)) {
continue;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
index 42079d7..c18966f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
@@ -44,8 +44,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlow
import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
@@ -220,7 +219,7 @@ class GenericEntityReader extends TimelineEntityReader {
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO, true);
- if (checkIsRelatedTo && !TimelineReaderUtils.matchRelations(
+ if (checkIsRelatedTo && !TimelineStorageUtils.matchRelations(
entity.getIsRelatedToEntities(), isRelatedTo)) {
return null;
}
@@ -235,7 +234,7 @@ class GenericEntityReader extends TimelineEntityReader {
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
- if (checkRelatesTo && !TimelineReaderUtils.matchRelations(
+ if (checkRelatesTo && !TimelineStorageUtils.matchRelations(
entity.getRelatesToEntities(), relatesTo)) {
return null;
}
@@ -251,7 +250,7 @@ class GenericEntityReader extends TimelineEntityReader {
fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
if (checkInfo &&
- !TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -265,7 +264,7 @@ class GenericEntityReader extends TimelineEntityReader {
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
- if (checkConfigs && !TimelineReaderUtils.matchFilters(
+ if (checkConfigs && !TimelineStorageUtils.matchFilters(
entity.getConfigs(), configFilters)) {
return null;
}
@@ -280,7 +279,7 @@ class GenericEntityReader extends TimelineEntityReader {
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
readEvents(entity, result, false);
- if (checkEvents && !TimelineReaderUtils.matchEventFilters(
+ if (checkEvents && !TimelineStorageUtils.matchEventFilters(
entity.getEvents(), eventFilters)) {
return null;
}
@@ -295,7 +294,7 @@ class GenericEntityReader extends TimelineEntityReader {
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
readMetrics(entity, result, EntityColumnPrefix.METRIC);
- if (checkMetrics && !TimelineReaderUtils.matchMetricFilters(
+ if (checkMetrics && !TimelineStorageUtils.matchMetricFilters(
entity.getMetrics(), metricFilters)) {
return null;
}
@@ -365,7 +364,7 @@ class GenericEntityReader extends TimelineEntityReader {
// the column name is of the form "eventId=timestamp=infoKey"
if (karr.length == 3) {
String id = Bytes.toString(karr[0]);
- long ts = TimelineWriterUtils.invert(Bytes.toLong(karr[1]));
+ long ts = TimelineStorageUtils.invertLong(Bytes.toLong(karr[1]));
String key = Separator.VALUES.joinEncoded(id, Long.toString(ts));
TimelineEvent event = eventsMap.get(key);
if (event == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 7c4a5da..3649865 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -46,7 +46,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlow
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
@@ -125,7 +125,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// if the entity is the application, the destination is the application
// table
- boolean isApplication = TimelineWriterUtils.isApplicationEntity(te);
+ boolean isApplication = TimelineStorageUtils.isApplicationEntity(te);
byte[] rowKey = isApplication ?
ApplicationRowKey.getRowKey(clusterId, userId, flowName, flowRunId,
appId) :
@@ -139,7 +139,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
storeRelations(rowKey, te, isApplication);
if (isApplication) {
- if (TimelineWriterUtils.isApplicationCreated(te)) {
+ if (TimelineStorageUtils.isApplicationCreated(te)) {
onApplicationCreated(clusterId, userId, flowName, flowVersion,
flowRunId, appId, te);
}
@@ -149,7 +149,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// if application has finished, store it's finish time and write final
// values
// of all metrics
- if (TimelineWriterUtils.isApplicationFinished(te)) {
+ if (TimelineStorageUtils.isApplicationFinished(te)) {
onApplicationFinished(clusterId, userId, flowName, flowVersion,
flowRunId, appId, te);
}
@@ -234,7 +234,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
Attribute attributeAppId = AggregationCompactionDimension.APPLICATION_ID
.getAttribute(appId);
FlowRunColumn.MAX_END_TIME.store(rowKey, flowRunTable, null,
- TimelineWriterUtils.getApplicationFinishedTime(te), attributeAppId);
+ TimelineStorageUtils.getApplicationFinishedTime(te), attributeAppId);
// store the final value of metrics since application has finished
Set<TimelineMetric> metrics = te.getMetrics();
@@ -406,9 +406,9 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
}
byte[] columnQualifierFirst =
Bytes.toBytes(Separator.VALUES.encode(eventId));
- byte[] columnQualifierWithTsBytes =
- Separator.VALUES.join(columnQualifierFirst,
- Bytes.toBytes(TimelineWriterUtils.invert(eventTimestamp)));
+ byte[] columnQualifierWithTsBytes = Separator.VALUES.
+ join(columnQualifierFirst, Bytes.toBytes(
+ TimelineStorageUtils.invertLong(eventTimestamp)));
Map<String, Object> eventInfo = event.getInfo();
if ((eventInfo == null) || (eventInfo.size() == 0)) {
// add separator since event key is empty
@@ -418,11 +418,11 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
if (isApplication) {
ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable,
compoundColumnQualifierBytes, null,
- TimelineWriterUtils.EMPTY_BYTES);
+ TimelineStorageUtils.EMPTY_BYTES);
} else {
EntityColumnPrefix.EVENT.store(rowKey, entityTable,
compoundColumnQualifierBytes, null,
- TimelineWriterUtils.EMPTY_BYTES);
+ TimelineStorageUtils.EMPTY_BYTES);
}
} else {
for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
index 10e3c2e..1cf6145 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.application;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
/**
* Represents a rowkey for the application table.
@@ -90,7 +90,7 @@ public class ApplicationRowKey {
String flowId, Long flowRunId) {
byte[] first = Bytes.toBytes(
Separator.QUALIFIERS.joinEncoded(clusterId, userId, flowId));
- byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
+ byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
return Separator.QUALIFIERS.join(first, second, new byte[0]);
}
@@ -112,8 +112,8 @@ public class ApplicationRowKey {
flowId));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
- byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
- byte[] third = Bytes.toBytes(appId);
+ byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
+ byte[] third = TimelineStorageUtils.encodeAppId(appId);
return Separator.QUALIFIERS.join(first, second, third);
}
@@ -135,9 +135,8 @@ public class ApplicationRowKey {
String flowId =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[2]));
long flowRunId =
- TimelineWriterUtils.invert(Bytes.toLong(rowKeyComponents[3]));
- String appId =
- Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[4]));
+ TimelineStorageUtils.invertLong(Bytes.toLong(rowKeyComponents[3]));
+ String appId = TimelineStorageUtils.decodeAppId(rowKeyComponents[4]);
return new ApplicationRowKey(clusterId, userId, flowId, flowRunId, appId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
index ca88056..133952e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
/**
* Represents a rowkey for the app_flow table.
@@ -49,7 +50,9 @@ public class AppToFlowRowKey {
* @return byte array with the row key
*/
public static byte[] getRowKey(String clusterId, String appId) {
- return Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId, appId));
+ byte[] first = Bytes.toBytes(clusterId);
+ byte[] second = TimelineStorageUtils.encodeAppId(appId);
+ return Separator.QUALIFIERS.join(first, second);
}
/**
@@ -64,7 +67,7 @@ public class AppToFlowRowKey {
}
String clusterId = Bytes.toString(rowKeyComponents[0]);
- String appId = Bytes.toString(rowKeyComponents[1]);
+ String appId = TimelineStorageUtils.decodeAppId(rowKeyComponents[1]);
return new AppToFlowRowKey(clusterId, appId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
index 9f91af8..1e82494 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
@@ -304,7 +304,7 @@ public enum Separator {
* @return source split by this separator.
*/
public byte[][] split(byte[] source, int limit) {
- return TimelineWriterUtils.split(source, this.bytes, limit);
+ return TimelineStorageUtils.split(source, this.bytes, limit);
}
/**
@@ -315,6 +315,6 @@ public enum Separator {
* @return source split by this separator.
*/
public byte[][] split(byte[] source) {
- return TimelineWriterUtils.split(source, this.bytes);
+ return TimelineStorageUtils.split(source, this.bytes);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java
deleted file mode 100644
index 91d7ba4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-
-public class TimelineReaderUtils {
- /**
- *
- * @param entityRelations the relations of an entity
- * @param relationFilters the relations for filtering
- * @return a boolean flag to indicate if both match
- */
- public static boolean matchRelations(
- Map<String, Set<String>> entityRelations,
- Map<String, Set<String>> relationFilters) {
- for (Map.Entry<String, Set<String>> relation : relationFilters.entrySet()) {
- Set<String> ids = entityRelations.get(relation.getKey());
- if (ids == null) {
- return false;
- }
- for (String id : relation.getValue()) {
- if (!ids.contains(id)) {
- return false;
- }
- }
- }
- return true;
- }
-
- /**
- *
- * @param map the map of key/value pairs in an entity
- * @param filters the map of key/value pairs for filtering
- * @return a boolean flag to indicate if both match
- */
- public static boolean matchFilters(Map<String, ? extends Object> map,
- Map<String, ? extends Object> filters) {
- for (Map.Entry<String, ? extends Object> filter : filters.entrySet()) {
- Object value = map.get(filter.getKey());
- if (value == null) {
- return false;
- }
- if (!value.equals(filter.getValue())) {
- return false;
- }
- }
- return true;
- }
-
- /**
- *
- * @param entityEvents the set of event objects in an entity
- * @param eventFilters the set of event Ids for filtering
- * @return a boolean flag to indicate if both match
- */
- public static boolean matchEventFilters(Set<TimelineEvent> entityEvents,
- Set<String> eventFilters) {
- Set<String> eventIds = new HashSet<String>();
- for (TimelineEvent event : entityEvents) {
- eventIds.add(event.getId());
- }
- for (String eventFilter : eventFilters) {
- if (!eventIds.contains(eventFilter)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- *
- * @param metrics the set of metric objects in an entity
- * @param metricFilters the set of metric Ids for filtering
- * @return a boolean flag to indicate if both match
- */
- public static boolean matchMetricFilters(Set<TimelineMetric> metrics,
- Set<String> metricFilters) {
- Set<String> metricIds = new HashSet<String>();
- for (TimelineMetric metric : metrics) {
- metricIds.add(metric.getId());
- }
-
- for (String metricFilter : metricFilters) {
- if (!metricIds.contains(metricFilter)) {
- return false;
- }
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
new file mode 100644
index 0000000..c1aaf19
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
@@ -0,0 +1,475 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+/**
+ * A bunch of utility functions used across TimelineReader and TimelineWriter.
+ */
+@Public
+@Unstable
+public class TimelineStorageUtils {
+
+ /** empty bytes */
+ public static final byte[] EMPTY_BYTES = new byte[0];
+
+ /** indicator for no limits for splitting */
+ public static final int NO_LIMIT_SPLIT = -1;
+
+ /** milliseconds in one day */
+ public static final long MILLIS_ONE_DAY = 86400000L;
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments. To identify the split
+ * ranges without the array copies, see {@link #splitRanges(byte[], byte[])}.
+ *
+ * @param source
+ * @param separator
+ * @return byte[] array after splitting the source
+ */
+ public static byte[][] split(byte[] source, byte[] separator) {
+ return split(source, separator, NO_LIMIT_SPLIT);
+ }
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments. To identify the split
+ * ranges without the array copies, see {@link #splitRanges(byte[], byte[])}.
+ *
+ * @param source
+ * @param separator
+ * @param limit a non-positive value indicates no limit on number of segments.
+ * @return byte[][] after splitting the input source
+ */
+ public static byte[][] split(byte[] source, byte[] separator, int limit) {
+ List<Range> segments = splitRanges(source, separator, limit);
+
+ byte[][] splits = new byte[segments.size()][];
+ for (int i = 0; i < segments.size(); i++) {
+ Range r = segments.get(i);
+ byte[] tmp = new byte[r.length()];
+ if (tmp.length > 0) {
+ System.arraycopy(source, r.start(), tmp, 0, r.length());
+ }
+ splits[i] = tmp;
+ }
+ return splits;
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ */
+ public static List<Range> splitRanges(byte[] source, byte[] separator) {
+ return splitRanges(source, separator, NO_LIMIT_SPLIT);
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ *
+ * @param source the source data
+ * @param separator the separator pattern to look for
+ * @param limit the maximum number of splits to identify in the source
+ */
+ public static List<Range> splitRanges(byte[] source, byte[] separator,
+ int limit) {
+ List<Range> segments = new ArrayList<Range>();
+ if ((source == null) || (separator == null)) {
+ return segments;
+ }
+ int start = 0;
+ itersource: for (int i = 0; i < source.length; i++) {
+ for (int j = 0; j < separator.length; j++) {
+ if (source[i + j] != separator[j]) {
+ continue itersource;
+ }
+ }
+ // all separator elements matched
+ if (limit > 0 && segments.size() >= (limit - 1)) {
+ // everything else goes in one final segment
+ break;
+ }
+ segments.add(new Range(start, i));
+ start = i + separator.length;
+ // i will be incremented again in outer for loop
+ i += separator.length - 1;
+ }
+ // add in remaining to a final range
+ if (start <= source.length) {
+ segments.add(new Range(start, source.length));
+ }
+ return segments;
+ }
+
+ /**
+ * Converts a timestamp into it's inverse timestamp to be used in (row) keys
+ * where we want to have the most recent timestamp in the top of the table
+ * (scans start at the most recent timestamp first).
+ *
+ * @param key value to be inverted so that the latest version will be first in
+ * a scan.
+ * @return inverted long
+ */
+ public static long invertLong(long key) {
+ return Long.MAX_VALUE - key;
+ }
+
+ /**
+ * Converts an int into it's inverse int to be used in (row) keys
+ * where we want to have the largest int value in the top of the table
+ * (scans start at the largest int first).
+ *
+ * @param key value to be inverted so that the latest version will be first in
+ * a scan.
+ * @return inverted int
+ */
+ public static int invertInt(int key) {
+ return Integer.MAX_VALUE - key;
+ }
+
+
+ /**
+ * Converts/encodes a string app Id into a byte representation for (row) keys.
+ * For conversion, we extract cluster timestamp and sequence id from the
+ * string app id (calls {@link ConverterUtils#toApplicationId(String)} for
+ * conversion) and then store it in a byte array of length 12 (8 bytes (long)
+ * for cluster timestamp followed 4 bytes(int) for sequence id). Both cluster
+ * timestamp and sequence id are inverted so that the most recent cluster
+ * timestamp and highest sequence id appears first in the table (i.e.
+ * application id appears in a descending order).
+ *
+ * @param appIdStr application id in string format i.e.
+ * application_{cluster timestamp}_{sequence id with min 4 digits}
+ *
+ * @return encoded byte representation of app id.
+ */
+ public static byte[] encodeAppId(String appIdStr) {
+ ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
+ byte[] appIdBytes = new byte[Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT];
+ byte[] clusterTs = Bytes.toBytes(invertLong(appId.getClusterTimestamp()));
+ System.arraycopy(clusterTs, 0, appIdBytes, 0, Bytes.SIZEOF_LONG);
+ byte[] seqId = Bytes.toBytes(invertInt(appId.getId()));
+ System.arraycopy(seqId, 0, appIdBytes, Bytes.SIZEOF_LONG, Bytes.SIZEOF_INT);
+ return appIdBytes;
+ }
+
+ /**
+ * Converts/decodes a 12 byte representation of app id for (row) keys to an
+ * app id in string format which can be returned back to client.
+ * For decoding, 12 bytes are interpreted as 8 bytes of inverted cluster
+ * timestamp(long) followed by 4 bytes of inverted sequence id(int). Calls
+ * {@link ApplicationId#toString} to generate string representation of app id.
+ *
+ * @param appIdBytes application id in byte representation.
+ *
+ * @return decoded app id in string format.
+ */
+ public static String decodeAppId(byte[] appIdBytes) {
+ if (appIdBytes.length != (Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT)) {
+ throw new IllegalArgumentException("Invalid app id in byte format");
+ }
+ long clusterTs = invertLong(Bytes.toLong(appIdBytes, 0, Bytes.SIZEOF_LONG));
+ int seqId =
+ invertInt(Bytes.toInt(appIdBytes, Bytes.SIZEOF_LONG, Bytes.SIZEOF_INT));
+ return ApplicationId.newInstance(clusterTs, seqId).toString();
+ }
+
+ /**
+ * returns the timestamp of that day's start (which is midnight 00:00:00 AM)
+ * for a given input timestamp
+ *
+ * @param ts
+ * @return timestamp of that day's beginning (midnight)
+ */
+ public static long getTopOfTheDayTimestamp(long ts) {
+ long dayTimestamp = ts - (ts % MILLIS_ONE_DAY);
+ return dayTimestamp;
+ }
+
+ /**
+ * Combines the input array of attributes and the input aggregation operation
+ * into a new array of attributes.
+ *
+ * @param attributes
+ * @param aggOp
+ * @return array of combined attributes
+ */
+ public static Attribute[] combineAttributes(Attribute[] attributes,
+ AggregationOperation aggOp) {
+ int newLength = getNewLengthCombinedAttributes(attributes, aggOp);
+ Attribute[] combinedAttributes = new Attribute[newLength];
+
+ if (attributes != null) {
+ System.arraycopy(attributes, 0, combinedAttributes, 0, attributes.length);
+ }
+
+ if (aggOp != null) {
+ Attribute a2 = aggOp.getAttribute();
+ combinedAttributes[newLength - 1] = a2;
+ }
+ return combinedAttributes;
+ }
+
+ /**
+ * Returns a number for the new array size. The new array is the combination
+ * of input array of attributes and the input aggregation operation.
+ *
+ * @param attributes
+ * @param aggOp
+ * @return the size for the new array
+ */
+ private static int getNewLengthCombinedAttributes(Attribute[] attributes,
+ AggregationOperation aggOp) {
+ int oldLength = getAttributesLength(attributes);
+ int aggLength = getAppOpLength(aggOp);
+ return oldLength + aggLength;
+ }
+
+ private static int getAppOpLength(AggregationOperation aggOp) {
+ if (aggOp != null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ private static int getAttributesLength(Attribute[] attributes) {
+ if (attributes != null) {
+ return attributes.length;
+ }
+ return 0;
+ }
+
+ /**
+ * checks if an application has finished
+ *
+ * @param te
+ * @return true if application has finished else false
+ */
+ public static boolean isApplicationFinished(TimelineEntity te) {
+ SortedSet<TimelineEvent> allEvents = te.getEvents();
+ if ((allEvents != null) && (allEvents.size() > 0)) {
+ TimelineEvent event = allEvents.last();
+ if (event.getId().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * get the time at which an app finished
+ *
+ * @param te
+ * @return true if application has finished else false
+ */
+ public static long getApplicationFinishedTime(TimelineEntity te) {
+ SortedSet<TimelineEvent> allEvents = te.getEvents();
+ if ((allEvents != null) && (allEvents.size() > 0)) {
+ TimelineEvent event = allEvents.last();
+ if (event.getId().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
+ return event.getTimestamp();
+ }
+ }
+ return 0l;
+ }
+
+ /**
+ * Checks if the input TimelineEntity object is an ApplicationEntity.
+ *
+ * @param te
+ * @return true if input is an ApplicationEntity, false otherwise
+ */
+ public static boolean isApplicationEntity(TimelineEntity te) {
+ return te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString());
+ }
+
+ /**
+ * Checks for the APPLICATION_CREATED event.
+ *
+ * @param te
+ * @return true is application event exists, false otherwise
+ */
+ public static boolean isApplicationCreated(TimelineEntity te) {
+ if (isApplicationEntity(te)) {
+ for (TimelineEvent event : te.getEvents()) {
+ if (event.getId()
+ .equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns the first seen aggregation operation as seen in the list of input
+ * tags or null otherwise
+ *
+ * @param tags
+ * @return AggregationOperation
+ */
+ public static AggregationOperation getAggregationOperationFromTagsList(
+ List<Tag> tags) {
+ for (AggregationOperation aggOp : AggregationOperation.values()) {
+ for (Tag tag : tags) {
+ if (tag.getType() == aggOp.getTagType()) {
+ return aggOp;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Creates a {@link Tag} from the input attribute.
+ *
+ * @param attribute
+ * @return Tag
+ */
+ public static Tag getTagFromAttribute(Entry<String, byte[]> attribute) {
+ // attribute could be either an Aggregation Operation or
+ // an Aggregation Dimension
+ // Get the Tag type from either
+ AggregationOperation aggOp = AggregationOperation
+ .getAggregationOperation(attribute.getKey());
+ if (aggOp != null) {
+ Tag t = new Tag(aggOp.getTagType(), attribute.getValue());
+ return t;
+ }
+
+ AggregationCompactionDimension aggCompactDim = AggregationCompactionDimension
+ .getAggregationCompactionDimension(attribute.getKey());
+ if (aggCompactDim != null) {
+ Tag t = new Tag(aggCompactDim.getTagType(), attribute.getValue());
+ return t;
+ }
+ return null;
+ }
+
+ /**
+ *
+ * @param entityRelations the relations of an entity
+ * @param relationFilters the relations for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchRelations(
+ Map<String, Set<String>> entityRelations,
+ Map<String, Set<String>> relationFilters) {
+ for (Map.Entry<String, Set<String>> relation : relationFilters.entrySet()) {
+ Set<String> ids = entityRelations.get(relation.getKey());
+ if (ids == null) {
+ return false;
+ }
+ for (String id : relation.getValue()) {
+ if (!ids.contains(id)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ *
+ * @param map the map of key/value pairs in an entity
+ * @param filters the map of key/value pairs for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchFilters(Map<String, ? extends Object> map,
+ Map<String, ? extends Object> filters) {
+ for (Map.Entry<String, ? extends Object> filter : filters.entrySet()) {
+ Object value = map.get(filter.getKey());
+ if (value == null) {
+ return false;
+ }
+ if (!value.equals(filter.getValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ *
+ * @param entityEvents the set of event objects in an entity
+ * @param eventFilters the set of event Ids for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchEventFilters(Set<TimelineEvent> entityEvents,
+ Set<String> eventFilters) {
+ Set<String> eventIds = new HashSet<String>();
+ for (TimelineEvent event : entityEvents) {
+ eventIds.add(event.getId());
+ }
+ for (String eventFilter : eventFilters) {
+ if (!eventIds.contains(eventFilter)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ *
+ * @param metrics the set of metric objects in an entity
+ * @param metricFilters the set of metric Ids for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchMetricFilters(Set<TimelineMetric> metrics,
+ Set<String> metricFilters) {
+ Set<String> metricIds = new HashSet<String>();
+ for (TimelineMetric metric : metrics) {
+ metricIds.add(metric.getId());
+ }
+
+ for (String metricFilter : metricFilters) {
+ if (!metricIds.contains(metricFilter)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
deleted file mode 100644
index 371371a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.SortedSet;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.Tag;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-/**
- * bunch of utility functions used across TimelineWriter classes
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class TimelineWriterUtils {
-
- /** empty bytes */
- public static final byte[] EMPTY_BYTES = new byte[0];
-
- /** indicator for no limits for splitting */
- public static final int NO_LIMIT_SPLIT = -1;
-
- /** milliseconds in one day */
- public static final long MILLIS_ONE_DAY = 86400000L;
-
- /**
- * Splits the source array into multiple array segments using the given
- * separator, up to a maximum of count items. This will naturally produce
- * copied byte arrays for each of the split segments. To identify the split
- * ranges without the array copies, see
- * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
- *
- * @param source
- * @param separator
- * @return byte[] array after splitting the source
- */
- public static byte[][] split(byte[] source, byte[] separator) {
- return split(source, separator, NO_LIMIT_SPLIT);
- }
-
- /**
- * Splits the source array into multiple array segments using the given
- * separator, up to a maximum of count items. This will naturally produce
- * copied byte arrays for each of the split segments. To identify the split
- * ranges without the array copies, see
- * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
- *
- * @param source
- * @param separator
- * @param limit a non-positive value indicates no limit on number of segments.
- * @return byte[][] after splitting the input source
- */
- public static byte[][] split(byte[] source, byte[] separator, int limit) {
- List<Range> segments = splitRanges(source, separator, limit);
-
- byte[][] splits = new byte[segments.size()][];
- for (int i = 0; i < segments.size(); i++) {
- Range r = segments.get(i);
- byte[] tmp = new byte[r.length()];
- if (tmp.length > 0) {
- System.arraycopy(source, r.start(), tmp, 0, r.length());
- }
- splits[i] = tmp;
- }
- return splits;
- }
-
- /**
- * Returns a list of ranges identifying [start, end) -- closed, open --
- * positions within the source byte array that would be split using the
- * separator byte array.
- */
- public static List<Range> splitRanges(byte[] source, byte[] separator) {
- return splitRanges(source, separator, NO_LIMIT_SPLIT);
- }
-
- /**
- * Returns a list of ranges identifying [start, end) -- closed, open --
- * positions within the source byte array that would be split using the
- * separator byte array.
- *
- * @param source the source data
- * @param separator the separator pattern to look for
- * @param limit the maximum number of splits to identify in the source
- */
- public static List<Range> splitRanges(byte[] source, byte[] separator,
- int limit) {
- List<Range> segments = new ArrayList<Range>();
- if ((source == null) || (separator == null)) {
- return segments;
- }
- int start = 0;
- itersource: for (int i = 0; i < source.length; i++) {
- for (int j = 0; j < separator.length; j++) {
- if (source[i + j] != separator[j]) {
- continue itersource;
- }
- }
- // all separator elements matched
- if (limit > 0 && segments.size() >= (limit - 1)) {
- // everything else goes in one final segment
- break;
- }
-
- segments.add(new Range(start, i));
- start = i + separator.length;
- // i will be incremented again in outer for loop
- i += separator.length - 1;
- }
- // add in remaining to a final range
- if (start <= source.length) {
- segments.add(new Range(start, source.length));
- }
- return segments;
- }
-
- /**
- * Converts a timestamp into it's inverse timestamp to be used in (row) keys
- * where we want to have the most recent timestamp in the top of the table
- * (scans start at the most recent timestamp first).
- *
- * @param key value to be inverted so that the latest version will be first in
- * a scan.
- * @return inverted long
- */
- public static long invert(Long key) {
- return Long.MAX_VALUE - key;
- }
-
- /**
- * returns the timestamp of that day's start (which is midnight 00:00:00 AM)
- * for a given input timestamp
- *
- * @param ts
- * @return timestamp of that day's beginning (midnight)
- */
- public static long getTopOfTheDayTimestamp(long ts) {
- long dayTimestamp = ts - (ts % MILLIS_ONE_DAY);
- return dayTimestamp;
- }
-
- /**
- * Combines the input array of attributes and the input aggregation operation
- * into a new array of attributes.
- *
- * @param attributes
- * @param aggOp
- * @return array of combined attributes
- */
- public static Attribute[] combineAttributes(Attribute[] attributes,
- AggregationOperation aggOp) {
- int newLength = getNewLengthCombinedAttributes(attributes, aggOp);
- Attribute[] combinedAttributes = new Attribute[newLength];
-
- if (attributes != null) {
- System.arraycopy(attributes, 0, combinedAttributes, 0, attributes.length);
- }
-
- if (aggOp != null) {
- Attribute a2 = aggOp.getAttribute();
- combinedAttributes[newLength - 1] = a2;
- }
- return combinedAttributes;
- }
-
- /**
- * Returns a number for the new array size. The new array is the combination
- * of input array of attributes and the input aggregation operation.
- *
- * @param attributes
- * @param aggOp
- * @return the size for the new array
- */
- private static int getNewLengthCombinedAttributes(Attribute[] attributes,
- AggregationOperation aggOp) {
- int oldLength = getAttributesLength(attributes);
- int aggLength = getAppOpLength(aggOp);
- return oldLength + aggLength;
- }
-
- private static int getAppOpLength(AggregationOperation aggOp) {
- if (aggOp != null) {
- return 1;
- }
- return 0;
- }
-
- private static int getAttributesLength(Attribute[] attributes) {
- if (attributes != null) {
- return attributes.length;
- }
- return 0;
- }
-
- /**
- * checks if an application has finished
- *
- * @param te
- * @return true if application has finished else false
- */
- public static boolean isApplicationFinished(TimelineEntity te) {
- SortedSet<TimelineEvent> allEvents = te.getEvents();
- if ((allEvents != null) && (allEvents.size() > 0)) {
- TimelineEvent event = allEvents.last();
- if (event.getId().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * get the time at which an app finished
- *
- * @param te
- * @return true if application has finished else false
- */
- public static long getApplicationFinishedTime(TimelineEntity te) {
- SortedSet<TimelineEvent> allEvents = te.getEvents();
- if ((allEvents != null) && (allEvents.size() > 0)) {
- TimelineEvent event = allEvents.last();
- if (event.getId().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
- return event.getTimestamp();
- }
- }
- return 0l;
- }
-
- /**
- * Checks if the input TimelineEntity object is an ApplicationEntity.
- *
- * @param te
- * @return true if input is an ApplicationEntity, false otherwise
- */
- public static boolean isApplicationEntity(TimelineEntity te) {
- return te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString());
- }
-
- /**
- * Checks for the APPLICATION_CREATED event.
- *
- * @param te
- * @return true is application event exists, false otherwise
- */
- public static boolean isApplicationCreated(TimelineEntity te) {
- if (isApplicationEntity(te)) {
- for (TimelineEvent event : te.getEvents()) {
- if (event.getId()
- .equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Returns the first seen aggregation operation as seen in the list of input
- * tags or null otherwise
- *
- * @param tags
- * @return AggregationOperation
- */
- public static AggregationOperation getAggregationOperationFromTagsList(
- List<Tag> tags) {
- for (AggregationOperation aggOp : AggregationOperation.values()) {
- for (Tag tag : tags) {
- if (tag.getType() == aggOp.getTagType()) {
- return aggOp;
- }
- }
- }
- return null;
- }
-
- /**
- * Creates a {@link Tag} from the input attribute.
- *
- * @param attribute
- * @return Tag
- */
- public static Tag getTagFromAttribute(Entry<String, byte[]> attribute) {
- // attribute could be either an Aggregation Operation or
- // an Aggregation Dimension
- // Get the Tag type from either
- AggregationOperation aggOp = AggregationOperation
- .getAggregationOperation(attribute.getKey());
- if (aggOp != null) {
- Tag t = new Tag(aggOp.getTagType(), attribute.getValue());
- return t;
- }
-
- AggregationCompactionDimension aggCompactDim = AggregationCompactionDimension
- .getAggregationCompactionDimension(attribute.getKey());
- if (aggCompactDim != null) {
- Tag t = new Tag(aggCompactDim.getTagType(), attribute.getValue());
- return t;
- }
- return null;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
index 6a534ed73..e0413c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
/**
* Represents a rowkey for the entity table.
@@ -90,9 +90,9 @@ public class EntityRowKey {
flowId));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
- byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
- byte[] third = Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId));
- return Separator.QUALIFIERS.join(first, second, third);
+ byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
+ byte[] third = TimelineStorageUtils.encodeAppId(appId);
+ return Separator.QUALIFIERS.join(first, second, third, new byte[0]);
}
/**
@@ -114,10 +114,11 @@ public class EntityRowKey {
flowId));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
- byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
- byte[] third =
- Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, entityType, ""));
- return Separator.QUALIFIERS.join(first, second, third);
+ byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
+ byte[] third = TimelineStorageUtils.encodeAppId(appId);
+ byte[] fourth =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(entityType, ""));
+ return Separator.QUALIFIERS.join(first, second, third, fourth);
}
/**
@@ -141,11 +142,11 @@ public class EntityRowKey {
flowId));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
- byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
- byte[] third =
- Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, entityType,
- entityId));
- return Separator.QUALIFIERS.join(first, second, third);
+ byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
+ byte[] third = TimelineStorageUtils.encodeAppId(appId);
+ byte[] fourth =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(entityType, entityId));
+ return Separator.QUALIFIERS.join(first, second, third, fourth);
}
/**
@@ -166,9 +167,8 @@ public class EntityRowKey {
String flowId =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[2]));
long flowRunId =
- TimelineWriterUtils.invert(Bytes.toLong(rowKeyComponents[3]));
- String appId =
- Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[4]));
+ TimelineStorageUtils.invertLong(Bytes.toLong(rowKeyComponents[3]));
+ String appId = TimelineStorageUtils.decodeAppId(rowKeyComponents[4]);
String entityType =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[5]));
String entityId =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
index b899e5c..38c0f3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
/**
@@ -114,7 +114,7 @@ public enum FlowActivityColumnPrefix implements ColumnPrefix<FlowActivityTable>
byte[] columnQualifier = ColumnHelper.getColumnQualifier(
this.columnPrefixBytes, qualifier);
- Attribute[] combinedAttributes = TimelineWriterUtils.combineAttributes(
+ Attribute[] combinedAttributes = TimelineStorageUtils.combineAttributes(
attributes, this.aggOp);
column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
combinedAttributes);
@@ -235,7 +235,7 @@ public enum FlowActivityColumnPrefix implements ColumnPrefix<FlowActivityTable>
byte[] columnQualifier = ColumnHelper.getColumnQualifier(
this.columnPrefixBytes, qualifier);
- Attribute[] combinedAttributes = TimelineWriterUtils.combineAttributes(
+ Attribute[] combinedAttributes = TimelineStorageUtils.combineAttributes(
attributes, this.aggOp);
column.store(rowKey, tableMutator, columnQualifier, null, inputValue,
combinedAttributes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
index 18ca599..f7841e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
/**
* Represents a rowkey for the flow activity table.
@@ -71,7 +71,7 @@ public class FlowActivityRowKey {
*/
public static byte[] getRowKey(String clusterId, String userId,
String flowId) {
- long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
+ long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
.currentTimeMillis());
return getRowKey(clusterId, dayTs, userId, flowId);
}
@@ -90,7 +90,7 @@ public class FlowActivityRowKey {
String flowId) {
return Separator.QUALIFIERS.join(
Bytes.toBytes(Separator.QUALIFIERS.encode(clusterId)),
- Bytes.toBytes(TimelineWriterUtils.invert(dayTs)),
+ Bytes.toBytes(TimelineStorageUtils.invertLong(dayTs)),
Bytes.toBytes(Separator.QUALIFIERS.encode(userId)),
Bytes.toBytes(Separator.QUALIFIERS.encode(flowId)));
}
@@ -108,7 +108,8 @@ public class FlowActivityRowKey {
String clusterId = Separator.QUALIFIERS.decode(Bytes
.toString(rowKeyComponents[0]));
- long dayTs = TimelineWriterUtils.invert(Bytes.toLong(rowKeyComponents[1]));
+ long dayTs =
+ TimelineStorageUtils.invertLong(Bytes.toLong(rowKeyComponents[1]));
String userId = Separator.QUALIFIERS.decode(Bytes
.toString(rowKeyComponents[2]));
String flowId = Separator.QUALIFIERS.decode(Bytes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
index ad30add..5079cc0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
/**
@@ -97,7 +97,7 @@ public enum FlowRunColumn implements Column<FlowRunTable> {
TypedBufferedMutator<FlowRunTable> tableMutator, Long timestamp,
Object inputValue, Attribute... attributes) throws IOException {
- Attribute[] combinedAttributes = TimelineWriterUtils.combineAttributes(
+ Attribute[] combinedAttributes = TimelineStorageUtils.combineAttributes(
attributes, aggOp);
column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
inputValue, combinedAttributes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
index d55f510..b090bba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
/**
@@ -112,7 +112,7 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
byte[] columnQualifier = ColumnHelper.getColumnQualifier(
this.columnPrefixBytes, qualifier);
- Attribute[] combinedAttributes = TimelineWriterUtils.combineAttributes(
+ Attribute[] combinedAttributes = TimelineStorageUtils.combineAttributes(
attributes, this.aggOp);
column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
combinedAttributes);
@@ -140,7 +140,7 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
byte[] columnQualifier = ColumnHelper.getColumnQualifier(
this.columnPrefixBytes, qualifier);
- Attribute[] combinedAttributes = TimelineWriterUtils.combineAttributes(
+ Attribute[] combinedAttributes = TimelineStorageUtils.combineAttributes(
attributes, this.aggOp);
column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
combinedAttributes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
index f743e5e..1984157 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
public class FlowRunCoprocessor extends BaseRegionObserver {
@@ -89,7 +89,7 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
List<Tag> tags = new ArrayList<>();
if ((attributes != null) && (attributes.size() > 0)) {
for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
- Tag t = TimelineWriterUtils.getTagFromAttribute(attribute);
+ Tag t = TimelineStorageUtils.getTagFromAttribute(attribute);
tags.add(t);
}
byte[] tagByteArray = Tag.fromList(tags);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
index 880d481..7ed3651 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
/**
* Represents a rowkey for the flow run table.
@@ -70,7 +70,7 @@ public class FlowRunRowKey {
userId, flowId));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
- byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
+ byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
return Separator.QUALIFIERS.join(first, second);
}
@@ -92,7 +92,7 @@ public class FlowRunRowKey {
String flowId =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[2]));
long flowRunId =
- TimelineWriterUtils.invert(Bytes.toLong(rowKeyComponents[3]));
+ TimelineStorageUtils.invertLong(Bytes.toLong(rowKeyComponents[3]));
return new FlowRunRowKey(clusterId, userId, flowId, flowRunId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
index 651bb3a..a537891 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
/**
* Invoked via the coprocessor when a Get or a Scan is issued for flow run
@@ -136,7 +136,7 @@ class FlowScanner implements RegionScanner, Closeable {
// So all cells in one qualifier come one after the other before we see the
// next column qualifier
ByteArrayComparator comp = new ByteArrayComparator();
- byte[] currentColumnQualifier = TimelineWriterUtils.EMPTY_BYTES;
+ byte[] currentColumnQualifier = TimelineStorageUtils.EMPTY_BYTES;
AggregationOperation currentAggOp = null;
SortedSet<Cell> currentColumnCells = new TreeSet<>(KeyValue.COMPARATOR);
Set<String> alreadySeenAggDim = new HashSet<>();
@@ -163,7 +163,7 @@ class FlowScanner implements RegionScanner, Closeable {
List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
cell.getTagsLength());
// We assume that all the operations for a particular column are the same
- return TimelineWriterUtils.getAggregationOperationFromTagsList(tags);
+ return TimelineStorageUtils.getAggregationOperationFromTagsList(tags);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index 3b0921b..701615e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -54,7 +54,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.application.Applica
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
@@ -366,7 +366,8 @@ public class TestHBaseTimelineStorage {
String flow = "some_flow_name";
String flowVersion = "AB7822C10F1111";
long runid = 1002345678919L;
- String appName = "some app name";
+ String appName =
+ ApplicationId.newInstance(System.currentTimeMillis(), 1).toString();
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
hbi.stop();
@@ -592,7 +593,8 @@ public class TestHBaseTimelineStorage {
byte[][] karr = (byte[][])e.getKey();
assertEquals(3, karr.length);
assertEquals(eventId, Bytes.toString(karr[0]));
- assertEquals(TimelineWriterUtils.invert(expTs), Bytes.toLong(karr[1]));
+ assertEquals(
+ TimelineStorageUtils.invertLong(expTs), Bytes.toLong(karr[1]));
assertEquals(expKey, Bytes.toString(karr[2]));
Object value = e.getValue();
// there should be only one timestamp and value
@@ -667,7 +669,8 @@ public class TestHBaseTimelineStorage {
String flow = "other_flow_name";
String flowVersion = "1111F01C2287BA";
long runid = 1009876543218L;
- String appName = "some app name";
+ String appName =
+ ApplicationId.newInstance(System.currentTimeMillis(), 1).toString();
byte[] startRow =
EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
@@ -700,7 +703,7 @@ public class TestHBaseTimelineStorage {
byte[][] karr = (byte[][])e.getKey();
assertEquals(3, karr.length);
assertEquals(eventId, Bytes.toString(karr[0]));
- assertEquals(TimelineWriterUtils.invert(expTs),
+ assertEquals(TimelineStorageUtils.invertLong(expTs),
Bytes.toLong(karr[1]));
// key must be empty
assertEquals(0, karr[2].length);
[05/50] [abbrv] hadoop git commit: YARN-3562. unit tests failures and
issues found from findbug from earlier ATS checkins (Naganarasimha G R via
sjlee)
Posted by sj...@apache.org.
YARN-3562. unit tests failures and issues found from findbug from earlier ATS checkins (Naganarasimha G R via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87823cf7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87823cf7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87823cf7
Branch: refs/heads/YARN-2928-rebase
Commit: 87823cf7cb961863283a7e7f330b5497e39585cc
Parents: 27d6778
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed May 6 20:31:50 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:48:30 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/ResourceTrackerService.java | 4 ++--
.../timelineservice/RMTimelineCollectorManager.java | 2 +-
.../yarn/server/resourcemanager/TestAppManager.java | 5 +++++
.../server/resourcemanager/TestClientRMService.java | 12 +++++++++++-
.../TestRMAppLogAggregationStatus.java | 4 ++++
.../metrics/TestSystemMetricsPublisher.java | 2 +-
.../resourcemanager/rmapp/TestRMAppTransitions.java | 5 +++++
.../org/apache/hadoop/yarn/server/MiniYARNCluster.java | 2 ++
.../storage/FileSystemTimelineWriterImpl.java | 7 +++++--
10 files changed, 39 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87823cf7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8a77216..02d2065 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -61,6 +61,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3551. Consolidate data model change according to the backend
implementation (Zhijie Shen via sjlee)
+ YARN-3562. unit tests failures and issues found from findbug from earlier
+ ATS checkins (Naganarasimha G R via sjlee)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87823cf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 59b9b5e..ad27648 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -544,8 +544,8 @@ public class ResourceTrackerService extends AbstractService implements
appId + " is not found in RMContext!");
} else {
String previousCollectorAddr = rmApp.getCollectorAddr();
- if (previousCollectorAddr == null ||
- previousCollectorAddr != collectorAddr) {
+ if (previousCollectorAddr == null
+ || !previousCollectorAddr.equals(collectorAddr)) {
// sending collector update event.
RMAppCollectorUpdateEvent event =
new RMAppCollectorUpdateEvent(appId, collectorAddr);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87823cf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
index 25e0e0f..7d1b657 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
@@ -65,7 +65,7 @@ public class RMTimelineCollectorManager extends TimelineCollectorManager {
break;
case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX:
collector.getTimelineEntityContext().setFlowRunId(
- Long.valueOf(parts[1]));
+ Long.parseLong(parts[1]));
break;
default:
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87823cf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index c435692..4ebb09d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -189,9 +190,11 @@ public class TestAppManager{
public int getCompletedAppsListSize() {
return super.getCompletedAppsListSize();
}
+
public int getCompletedAppsInStateStore() {
return this.completedAppsInStateStore;
}
+
public void submitApplication(
ApplicationSubmissionContext submissionContext, String user)
throws YarnException {
@@ -222,6 +225,8 @@ public class TestAppManager{
long now = System.currentTimeMillis();
rmContext = mockRMContext(1, now - 10);
+ rmContext
+ .setRMTimelineCollectorManager(mock(RMTimelineCollectorManager.class));
ResourceScheduler scheduler = mockResourceScheduler();
((RMContextImpl)rmContext).setScheduler(scheduler);
Configuration conf = new Configuration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87823cf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 49b5b55..8e3f0af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -24,6 +24,7 @@ import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@@ -136,6 +137,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
@@ -626,6 +628,9 @@ public class TestClientRMService {
new EventHandler<Event>() {
public void handle(Event event) {}
});
+ doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
+ .getRMTimelineCollectorManager();
+
ApplicationId appId1 = getApplicationId(100);
ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
@@ -713,6 +718,9 @@ public class TestClientRMService {
mockRMContext(yarnScheduler, rmContext);
RMStateStore stateStore = mock(RMStateStore.class);
when(rmContext.getStateStore()).thenReturn(stateStore);
+ doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
+ .getRMTimelineCollectorManager();
+
RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler,
null, mock(ApplicationACLsManager.class), new Configuration());
when(rmContext.getDispatcher().getEventHandler()).thenReturn(
@@ -884,7 +892,9 @@ public class TestClientRMService {
};
when(rmContext.getDispatcher().getEventHandler()).thenReturn(eventHandler);
-
+ doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
+ .getRMTimelineCollectorManager();
+
final ClientRMService rmService =
new ClientRMService(rmContext, yarnScheduler, appManager, null, null,
null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87823cf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
index 1f307aa..ec71846 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -91,6 +92,9 @@ public class TestRMAppLogAggregationStatus {
rmContext.setSystemMetricsPublisher(new SystemMetricsPublisher());
rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
+ rmContext
+ .setRMTimelineCollectorManager(mock(RMTimelineCollectorManager.class));
+
scheduler = mock(YarnScheduler.class);
doAnswer(
new Answer<Void>() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87823cf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 98daae7..5965ecb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -74,7 +74,7 @@ public class TestSystemMetricsPublisher {
public static void setup() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
MemoryTimelineStore.class, TimelineStore.class);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87823cf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index afe95cf..0e748a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
+
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.doReturn;
@@ -80,6 +81,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSec
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
@@ -225,6 +227,9 @@ public class TestRMAppTransitions {
.getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
doReturn(resourceScheduler).when(rmContext).getScheduler();
+ doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
+ .getRMTimelineCollectorManager();
+
rmDispatcher.register(RMAppAttemptEventType.class,
new TestApplicationAttemptEventDispatcher(this.rmContext));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87823cf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index a348390..42c2fd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -523,6 +523,8 @@ public class MiniYARNCluster extends CompositeService {
MiniYARNCluster.getHostname() + ":0");
config.set(YarnConfiguration.NM_LOCALIZER_ADDRESS,
MiniYARNCluster.getHostname() + ":0");
+ config.set(YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
+ MiniYARNCluster.getHostname() + ":0");
WebAppUtils
.setNMWebAppHostNameAndPort(config,
MiniYARNCluster.getHostname(), 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87823cf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index dd8ad06..ee1515d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -20,8 +20,9 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -83,7 +84,9 @@ public class FileSystemTimelineWriterImpl extends AbstractService
String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId,flowName,
flowVersion, String.valueOf(flowRun), appId, entity.getType());
String fileName = dir + entity.getId() + TIMELINE_SERVICE_STORAGE_EXTENSION;
- out = new PrintWriter(new BufferedWriter(new FileWriter(fileName, true)));
+ out =
+ new PrintWriter(new BufferedWriter(new OutputStreamWriter(
+ new FileOutputStream(fileName, true), "UTF-8")));
out.println(TimelineUtils.dumpTimelineRecordtoJSON(entity));
out.write("\n");
} catch (IOException ioe) {
[48/50] [abbrv] hadoop git commit: YARN-4178. [storage
implementation] app id as string in row keys can cause incorrect ordering
(Varun Saxena via sjlee)
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineStorageUtils.java
new file mode 100644
index 0000000..046eda1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineStorageUtils.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.junit.Test;
+
+public class TestTimelineStorageUtils {
+
+ @Test
+ public void testEncodeDecodeAppId() {
+ long currentTs = System.currentTimeMillis();
+ ApplicationId appId1 = ApplicationId.newInstance(currentTs, 1);
+ ApplicationId appId2 = ApplicationId.newInstance(currentTs, 2);
+ ApplicationId appId3 = ApplicationId.newInstance(currentTs + 300, 1);
+ String appIdStr1 = appId1.toString();
+ String appIdStr2 = appId2.toString();
+ String appIdStr3 = appId3.toString();
+ byte[] appIdBytes1 = TimelineStorageUtils.encodeAppId(appIdStr1);
+ byte[] appIdBytes2 = TimelineStorageUtils.encodeAppId(appIdStr2);
+ byte[] appIdBytes3 = TimelineStorageUtils.encodeAppId(appIdStr3);
+ // App ids' should be encoded in a manner wherein descending order
+ // is maintained.
+ assertTrue("Ordering of app ids' is incorrect",
+ Bytes.compareTo(appIdBytes1, appIdBytes2) > 0 &&
+ Bytes.compareTo(appIdBytes1, appIdBytes3) > 0 &&
+ Bytes.compareTo(appIdBytes2, appIdBytes3) > 0);
+ String decodedAppId1 = TimelineStorageUtils.decodeAppId(appIdBytes1);
+ String decodedAppId2 = TimelineStorageUtils.decodeAppId(appIdBytes2);
+ String decodedAppId3 = TimelineStorageUtils.decodeAppId(appIdBytes3);
+ assertTrue("Decoded app id is not same as the app id encoded",
+ appIdStr1.equals(decodedAppId1));
+ assertTrue("Decoded app id is not same as the app id encoded",
+ appIdStr2.equals(decodedAppId2));
+ assertTrue("Decoded app id is not same as the app id encoded",
+ appIdStr3.equals(decodedAppId3));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java
deleted file mode 100644
index 4f96f87..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.junit.Test;
-
-public class TestTimelineWriterUtils {
-
- @Test
- public void test() {
- // TODO: implement a test for the remaining method in TimelineWriterUtils.
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fec6f63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index 6bdec6b..c957dad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -49,7 +49,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReader
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -166,7 +166,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
assertEquals(flow, flowActivityRowKey.getFlowId());
- long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
+ long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
.currentTimeMillis());
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
assertEquals(1, values.size());
@@ -281,7 +281,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
assertEquals(flow, flowActivityRowKey.getFlowId());
- long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
+ long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
.currentTimeMillis());
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
assertEquals(1, values.size());
@@ -360,7 +360,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivity.getCluster());
assertEquals(user, flowActivity.getUser());
assertEquals(flow, flowActivity.getFlowName());
- long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
+ long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
.currentTimeMillis());
assertEquals(dayTs, flowActivity.getDate().getTime());
Set<FlowRunEntity> flowRuns = flowActivity.getFlowRuns();
@@ -410,7 +410,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
assertEquals(flow, flowActivityRowKey.getFlowId());
- long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
+ long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
.currentTimeMillis());
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
[34/50] [abbrv] hadoop git commit: YARN-4025. Deal with byte
representations of Longs in writer code. Contributed by Sangjin Lee and
Vrushali C.
Posted by sj...@apache.org.
YARN-4025. Deal with byte representations of Longs in writer code. Contributed by Sangjin Lee and Vrushali C.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f98f97b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f98f97b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f98f97b3
Branch: refs/heads/YARN-2928-rebase
Commit: f98f97b35139b7cc3d00537882a57ac0b53f701e
Parents: d3fb80b
Author: Junping Du <ju...@apache.org>
Authored: Wed Aug 19 10:00:33 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 16:13:29 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../storage/HBaseTimelineReaderImpl.java | 68 +++---
.../storage/HBaseTimelineWriterImpl.java | 20 +-
.../application/ApplicationColumnPrefix.java | 40 ++++
.../storage/application/ApplicationTable.java | 6 +-
.../storage/common/ColumnHelper.java | 99 ++++++++-
.../storage/common/Separator.java | 16 +-
.../storage/common/TimelineWriterUtils.java | 9 +-
.../storage/entity/EntityColumnPrefix.java | 40 ++++
.../storage/entity/EntityTable.java | 6 +-
.../storage/TestHBaseTimelineWriterImpl.java | 207 +++++++++++--------
11 files changed, 373 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e5c609a..d80ffa0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -94,6 +94,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3045. Implement NM writing container lifecycle events to Timeline
Service v2. (Naganarasimha G R via junping_du)
+ YARN-4025. Deal with byte representations of Longs in writer code.
+ (Sangjin Lee and Vrushali C via junping_du)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index 094f868..c514c20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -19,12 +19,9 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.Map;
import java.util.NavigableMap;
import java.util.NavigableSet;
@@ -431,44 +428,51 @@ public class HBaseTimelineReaderImpl
Map<String, Object> columns = prefix.readResults(result);
if (isConfig) {
for (Map.Entry<String, Object> column : columns.entrySet()) {
- entity.addConfig(column.getKey(), column.getKey().toString());
+ entity.addConfig(column.getKey(), column.getValue().toString());
}
} else {
entity.addInfo(columns);
}
}
+ /**
+ * Read events from the entity table or the application table. The column name
+ * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
+ * if there is no info associated with the event.
+ *
+ * See {@link EntityTable} and {@link ApplicationTable} for a more detailed
+ * schema description.
+ */
private static void readEvents(TimelineEntity entity, Result result,
boolean isApplication) throws IOException {
Map<String, TimelineEvent> eventsMap = new HashMap<>();
- Map<String, Object> eventsResult = isApplication ?
- ApplicationColumnPrefix.EVENT.readResults(result) :
- EntityColumnPrefix.EVENT.readResults(result);
- for (Map.Entry<String,Object> eventResult : eventsResult.entrySet()) {
- Collection<String> tokens =
- Separator.VALUES.splitEncoded(eventResult.getKey());
- if (tokens.size() != 2 && tokens.size() != 3) {
- throw new IOException(
- "Invalid event column name: " + eventResult.getKey());
- }
- Iterator<String> idItr = tokens.iterator();
- String id = idItr.next();
- String tsStr = idItr.next();
- // TODO: timestamp is not correct via ser/des through UTF-8 string
- Long ts =
- TimelineWriterUtils.invert(Bytes.toLong(tsStr.getBytes(
- StandardCharsets.UTF_8)));
- String key = Separator.VALUES.joinEncoded(id, ts.toString());
- TimelineEvent event = eventsMap.get(key);
- if (event == null) {
- event = new TimelineEvent();
- event.setId(id);
- event.setTimestamp(ts);
- eventsMap.put(key, event);
- }
- if (tokens.size() == 3) {
- String infoKey = idItr.next();
- event.addInfo(infoKey, eventResult.getValue());
+ Map<?, Object> eventsResult = isApplication ?
+ ApplicationColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result) :
+ EntityColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
+ for (Map.Entry<?, Object> eventResult : eventsResult.entrySet()) {
+ byte[][] karr = (byte[][])eventResult.getKey();
+ // the column name is of the form "eventId=timestamp=infoKey"
+ if (karr.length == 3) {
+ String id = Bytes.toString(karr[0]);
+ long ts = TimelineWriterUtils.invert(Bytes.toLong(karr[1]));
+ String key = Separator.VALUES.joinEncoded(id, Long.toString(ts));
+ TimelineEvent event = eventsMap.get(key);
+ if (event == null) {
+ event = new TimelineEvent();
+ event.setId(id);
+ event.setTimestamp(ts);
+ eventsMap.put(key, event);
+ }
+ // handle empty info
+ String infoKey = karr[2].length == 0 ? null : Bytes.toString(karr[2]);
+ if (infoKey != null) {
+ event.addInfo(infoKey, eventResult.getValue());
+ }
+ } else {
+ LOG.warn("incorrectly formatted column name: it will be discarded");
+ continue;
}
}
Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 96192cc..772002d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -300,25 +300,27 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
byte[] compoundColumnQualifierBytes =
Separator.VALUES.join(columnQualifierWithTsBytes,
null);
- String compoundColumnQualifier =
- Bytes.toString(compoundColumnQualifierBytes);
- EntityColumnPrefix.EVENT.store(rowKey, entityTable,
- compoundColumnQualifier, null, TimelineWriterUtils.EMPTY_BYTES);
+ if (isApplication) {
+ ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable,
+ compoundColumnQualifierBytes, null,
+ TimelineWriterUtils.EMPTY_BYTES);
+ } else {
+ EntityColumnPrefix.EVENT.store(rowKey, entityTable,
+ compoundColumnQualifierBytes, null,
+ TimelineWriterUtils.EMPTY_BYTES);
+ }
} else {
for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
// eventId?infoKey
byte[] compoundColumnQualifierBytes =
Separator.VALUES.join(columnQualifierWithTsBytes,
Bytes.toBytes(info.getKey()));
- // convert back to string to avoid additional API on store.
- String compoundColumnQualifier =
- Bytes.toString(compoundColumnQualifierBytes);
if (isApplication) {
ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable,
- compoundColumnQualifier, null, info.getValue());
+ compoundColumnQualifierBytes, null, info.getValue());
} else {
EntityColumnPrefix.EVENT.store(rowKey, entityTable,
- compoundColumnQualifier, null, info.getValue());
+ compoundColumnQualifierBytes, null, info.getValue());
}
} // for info: eventInfo
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
index cd9e845..ad1def6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
@@ -111,6 +111,31 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
* TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
*/
public void store(byte[] rowKey,
+ TypedBufferedMutator<ApplicationTable> tableMutator, byte[] qualifier,
+ Long timestamp, Object inputValue) throws IOException {
+
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ */
+ public void store(byte[] rowKey,
TypedBufferedMutator<ApplicationTable> tableMutator, String qualifier,
Long timestamp, Object inputValue) throws IOException {
@@ -150,6 +175,21 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
return column.readResults(result, columnPrefixBytes);
}
+ /**
+ * @param result from which to read columns
+ * @return the latest values of columns in the column family. The column
+ * qualifier is returned as a list of parts, each part a byte[]. This
+ * is to facilitate returning byte arrays of values that were not
+ * Strings. If they can be treated as Strings, you should use
+ * {@link #readResults(Result)} instead.
+ * @throws IOException
+ */
+ public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result)
+ throws IOException {
+ return column.readResultsHavingCompoundColumnQualifiers(result,
+ columnPrefixBytes);
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
index d2a2cb9..a997997 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
@@ -57,12 +57,12 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* | | infoValue | metricValue1 | |
* | | | @timestamp2 | |
* | | r!relatesToKey: | | |
- * | | id3?id4?id5 | | |
+ * | | id3=id4=id5 | | |
* | | | | |
* | | s!isRelatedToKey: | | |
- * | | id7?id9?id6 | | |
+ * | | id7=id9=id6 | | |
* | | | | |
- * | | e!eventId?timestamp?infoKey: | | |
+ * | | e!eventId=timestamp=infoKey: | | |
* | | eventInfoValue | | |
* | | | | |
* | | flowVersion: | | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
index a902924..f1b7c58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -24,6 +24,8 @@ import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.TreeMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
@@ -37,6 +39,7 @@ import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
* @param <T> refers to the table.
*/
public class ColumnHelper<T> {
+ private static final Log LOG = LogFactory.getLog(ColumnHelper.class);
private final ColumnFamily<T> columnFamily;
@@ -143,6 +146,7 @@ public class ColumnHelper<T> {
.entrySet()) {
String columnName = null;
if (columnPrefixBytes == null) {
+ LOG.info("null prefix was specified; returning all columns");
// Decode the spaces we encoded in the column name.
columnName = Separator.decode(entry.getKey(), Separator.SPACE);
} else {
@@ -181,32 +185,43 @@ public class ColumnHelper<T> {
/**
* @param result from which to read columns
* @param columnPrefixBytes optional prefix to limit columns. If null all
- * columns are returned.
- * @return the latest values of columns in the column family.
+ * columns are returned.
+ * @return the latest values of columns in the column family. This assumes
+ * that the column name parts are all Strings by default. If the
+ * column name parts should be treated natively and not be converted
+ * back and forth from Strings, you should use
+ * {@link #readResultsHavingCompoundColumnQualifiers(Result, byte[])}
+ * instead.
* @throws IOException
*/
- public Map<String, Object> readResults(Result result, byte[] columnPrefixBytes)
- throws IOException {
+ public Map<String, Object> readResults(Result result,
+ byte[] columnPrefixBytes) throws IOException {
Map<String, Object> results = new HashMap<String, Object>();
if (result != null) {
Map<byte[], byte[]> columns = result.getFamilyMap(columnFamilyBytes);
for (Entry<byte[], byte[]> entry : columns.entrySet()) {
- if (entry.getKey() != null && entry.getKey().length > 0) {
+ byte[] columnKey = entry.getKey();
+ if (columnKey != null && columnKey.length > 0) {
String columnName = null;
if (columnPrefixBytes == null) {
+ LOG.info("null prefix was specified; returning all columns");
// Decode the spaces we encoded in the column name.
- columnName = Separator.decode(entry.getKey(), Separator.SPACE);
+ columnName = Separator.decode(columnKey, Separator.SPACE);
} else {
// A non-null prefix means columns are actually of the form
// prefix!columnNameRemainder
byte[][] columnNameParts =
- Separator.QUALIFIERS.split(entry.getKey(), 2);
+ Separator.QUALIFIERS.split(columnKey, 2);
byte[] actualColumnPrefixBytes = columnNameParts[0];
if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
&& columnNameParts.length == 2) {
// This is the prefix that we want
+ // if the column name is a compound qualifier
+ // with non string datatypes, the following decode will not
+ // work correctly since it considers all components to be String
+ // invoke the readResultsHavingCompoundColumnQualifiers function
columnName = Separator.decode(columnNameParts[1]);
}
}
@@ -223,6 +238,56 @@ public class ColumnHelper<T> {
}
/**
+ * @param result from which to read columns
+ * @param columnPrefixBytes optional prefix to limit columns. If null all
+ * columns are returned.
+ * @return the latest values of columns in the column family. If the column
+ * prefix is null, the column qualifier is returned as Strings. For a
+ * non-null column prefix bytes, the column qualifier is returned as
+ * a list of parts, each part a byte[]. This is to facilitate
+ * returning byte arrays of values that were not Strings.
+ * @throws IOException
+ */
+ public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result,
+ byte[] columnPrefixBytes) throws IOException {
+ // handle the case where the column prefix is null
+ // it is the same as readResults() so simply delegate to that implementation
+ if (columnPrefixBytes == null) {
+ return readResults(result, null);
+ }
+
+ Map<byte[][], Object> results = new HashMap<byte[][], Object>();
+
+ if (result != null) {
+ Map<byte[], byte[]> columns = result.getFamilyMap(columnFamilyBytes);
+ for (Entry<byte[], byte[]> entry : columns.entrySet()) {
+ byte[] columnKey = entry.getKey();
+ if (columnKey != null && columnKey.length > 0) {
+ // A non-null prefix means columns are actually of the form
+ // prefix!columnNameRemainder
+ // with a compound column qualifier, we are presuming existence of a
+ // prefix
+ byte[][] columnNameParts = Separator.QUALIFIERS.split(columnKey, 2);
+ if (columnNameParts.length > 0) {
+ byte[] actualColumnPrefixBytes = columnNameParts[0];
+ if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
+ && columnNameParts.length == 2) {
+ // This is the prefix that we want
+ byte[][] columnQualifierParts =
+ Separator.VALUES.split(columnNameParts[1]);
+ Object value = GenericObjectMapper.read(entry.getValue());
+ // we return the columnQualifier in parts since we don't know
+ // which part is of which data type
+ results.put(columnQualifierParts, value);
+ }
+ }
+ }
+ } // for entry
+ }
+ return results;
+ }
+
+ /**
* @param columnPrefixBytes The byte representation for the column prefix.
* Should not contain {@link Separator#QUALIFIERS}.
* @param qualifier for the remainder of the column. Any
@@ -247,4 +312,24 @@ public class ColumnHelper<T> {
return columnQualifier;
}
+ /**
+ * @param columnPrefixBytes The byte representation for the column prefix.
+ * Should not contain {@link Separator#QUALIFIERS}.
+ * @param qualifier the byte representation for the remainder of the column.
+ * @return fully sanitized column qualifier that is a combination of prefix
+ * and qualifier. If prefix is null, the result is simply the encoded
+ * qualifier without any separator.
+ */
+ public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
+ byte[] qualifier) {
+
+ if (columnPrefixBytes == null) {
+ return qualifier;
+ }
+
+ byte[] columnQualifier =
+ Separator.QUALIFIERS.join(columnPrefixBytes, qualifier);
+ return columnQualifier;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
index 3319419..9f91af8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
@@ -37,7 +37,7 @@ public enum Separator {
/**
* separator in values, and/or compound key/column qualifier fields.
*/
- VALUES("?", "%1$"),
+ VALUES("=", "%1$"),
/**
* separator in values, often used to avoid having these in qualifiers and
@@ -299,12 +299,22 @@ public enum Separator {
* up to a maximum of count items. This will naturally produce copied byte
* arrays for each of the split segments.
* @param source to be split
- * @param limit on how many segments are supposed to be returned. Negative
- * value indicates no limit on number of segments.
+ * @param limit on how many segments are supposed to be returned. A
+ * non-positive value indicates no limit on number of segments.
* @return source split by this separator.
*/
public byte[][] split(byte[] source, int limit) {
return TimelineWriterUtils.split(source, this.bytes, limit);
}
+ /**
+ * Splits the source array into multiple array segments using this separator,
+ * as many times as splits are found. This will naturally produce copied byte
+ * arrays for each of the split segments.
+ * @param source to be split
+ * @return source split by this separator.
+ */
+ public byte[][] split(byte[] source) {
+ return TimelineWriterUtils.split(source, this.bytes);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
index c957bf5..58bdedc7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
@@ -33,6 +33,9 @@ public class TimelineWriterUtils {
/** empty bytes */
public static final byte[] EMPTY_BYTES = new byte[0];
+ /** indicator for no limits for splitting */
+ public static final int NO_LIMIT_SPLIT = -1;
+
/**
* Splits the source array into multiple array segments using the given
* separator, up to a maximum of count items. This will naturally produce
@@ -45,7 +48,7 @@ public class TimelineWriterUtils {
* @return byte[] array after splitting the source
*/
public static byte[][] split(byte[] source, byte[] separator) {
- return split(source, separator, -1);
+ return split(source, separator, NO_LIMIT_SPLIT);
}
/**
@@ -57,7 +60,7 @@ public class TimelineWriterUtils {
*
* @param source
* @param separator
- * @param limit a negative value indicates no limit on number of segments.
+ * @param limit a non-positive value indicates no limit on number of segments.
* @return byte[][] after splitting the input source
*/
public static byte[][] split(byte[] source, byte[] separator, int limit) {
@@ -81,7 +84,7 @@ public class TimelineWriterUtils {
* separator byte array.
*/
public static List<Range> splitRanges(byte[] source, byte[] separator) {
- return splitRanges(source, separator, -1);
+ return splitRanges(source, separator, NO_LIMIT_SPLIT);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index c8485c0..75ff742 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -131,6 +131,31 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
*
* @see
* org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ */
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<EntityTable> tableMutator, byte[] qualifier,
+ Long timestamp, Object inputValue) throws IOException {
+
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
* #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
*/
public Object readResult(Result result, String qualifier) throws IOException {
@@ -150,6 +175,21 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
return column.readResults(result, columnPrefixBytes);
}
+ /**
+ * @param result from which to read columns
+ * @return the latest values of columns in the column family. The column
+ * qualifier is returned as a list of parts, each part a byte[]. This
+ * is to facilitate returning byte arrays of values that were not
+ * Strings. If they can be treated as Strings, you should use
+ * {@link #readResults(Result)} instead.
+ * @throws IOException
+ */
+ public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result)
+ throws IOException {
+ return column.readResultsHavingCompoundColumnQualifiers(result,
+ columnPrefixBytes);
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 9a8bd8c..96773b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -58,12 +58,12 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* | | infoValue | | |
* | | | | |
* | | r!relatesToKey: | | |
- * | | id3?id4?id5 | | |
+ * | | id3=id4=id5 | | |
* | | | | |
* | | s!isRelatedToKey | | |
- * | | id7?id9?id6 | | |
+ * | | id7=id9=id6 | | |
* | | | | |
- * | | e!eventId?timestamp?infoKey: | | |
+ * | | e!eventId=timestamp=infoKey: | | |
* | | eventInfoValue | | |
* | | | | |
* | | flowVersion: | | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98f97b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index 95f88d1..2875e01 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -27,8 +27,8 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.NavigableMap;
+import java.util.NavigableSet;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
@@ -75,7 +76,7 @@ import org.junit.Test;
* even if other records exist in the table. Use a different cluster name if
* you add a new test.
*/
-public class TestHBaseTimelineWriterImpl {
+public class TestHBaseTimelineStorage {
private static HBaseTestingUtility util;
@@ -101,8 +102,8 @@ public class TestHBaseTimelineWriterImpl {
ApplicationEntity entity = new ApplicationEntity();
String id = "hello";
entity.setId(id);
- Long cTime = 1425016501000L;
- Long mTime = 1425026901000L;
+ long cTime = 1425016501000L;
+ long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
entity.setModifiedTime(mTime);
@@ -197,19 +198,16 @@ public class TestHBaseTimelineWriterImpl {
Number val =
(Number) ApplicationColumn.CREATED_TIME.readResult(result);
- Long cTime1 = val.longValue();
+ long cTime1 = val.longValue();
assertEquals(cTime1, cTime);
val = (Number) ApplicationColumn.MODIFIED_TIME.readResult(result);
- Long mTime1 = val.longValue();
+ long mTime1 = val.longValue();
assertEquals(mTime1, mTime);
Map<String, Object> infoColumns =
ApplicationColumnPrefix.INFO.readResults(result);
- assertEquals(infoMap.size(), infoColumns.size());
- for (String infoItem : infoMap.keySet()) {
- assertEquals(infoMap.get(infoItem), infoColumns.get(infoItem));
- }
+ assertEquals(infoMap, infoColumns);
// Remember isRelatedTo is of type Map<String, Set<String>>
for (String isRelatedToKey : isRelatedTo.keySet()) {
@@ -245,27 +243,15 @@ public class TestHBaseTimelineWriterImpl {
// Configuration
Map<String, Object> configColumns =
ApplicationColumnPrefix.CONFIG.readResults(result);
- assertEquals(conf.size(), configColumns.size());
- for (String configItem : conf.keySet()) {
- assertEquals(conf.get(configItem), configColumns.get(configItem));
- }
+ assertEquals(conf, configColumns);
NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
- // We got metrics back
- assertNotNull(metricMap);
- // Same number of metrics as we wrote
- assertEquals(metricValues.entrySet().size(), metricMap.entrySet().size());
-
- // Iterate over original metrics and confirm that they are present
- // here.
- for (Entry<Long, Number> metricEntry : metricValues.entrySet()) {
- assertEquals(metricEntry.getValue(),
- metricMap.get(metricEntry.getKey()));
- }
+ assertEquals(metricValues, metricMap);
+ // read the timeline entity using the reader this time
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, id,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
@@ -274,6 +260,31 @@ public class TestHBaseTimelineWriterImpl {
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
assertEquals(1, es1.size());
+
+ // verify attributes
+ assertEquals(id, e1.getId());
+ assertEquals(TimelineEntityType.YARN_APPLICATION.toString(),
+ e1.getType());
+ assertEquals(cTime, e1.getCreatedTime());
+ assertEquals(mTime, e1.getModifiedTime());
+ Map<String, Object> infoMap2 = e1.getInfo();
+ assertEquals(infoMap, infoMap2);
+
+ Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
+ assertEquals(isRelatedTo, isRelatedTo2);
+
+ Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
+ assertEquals(relatesTo, relatesTo2);
+
+ Map<String, String> conf2 = e1.getConfigs();
+ assertEquals(conf, conf2);
+
+ Set<TimelineMetric> metrics2 = e1.getMetrics();
+ assertEquals(metrics, metrics2);
+ for (TimelineMetric metric2 : metrics2) {
+ Map<Long, Number> metricValues2 = metric2.getValues();
+ assertEquals(metricValues, metricValues2);
+ }
} finally {
if (hbi != null) {
hbi.stop();
@@ -294,8 +305,8 @@ public class TestHBaseTimelineWriterImpl {
String type = "world";
entity.setId(id);
entity.setType(type);
- Long cTime = 1425016501000L;
- Long mTime = 1425026901000L;
+ long cTime = 1425016501000L;
+ long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
entity.setModifiedTime(mTime);
@@ -396,20 +407,16 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(type, type1);
Number val = (Number) EntityColumn.CREATED_TIME.readResult(result);
- Long cTime1 = val.longValue();
+ long cTime1 = val.longValue();
assertEquals(cTime1, cTime);
val = (Number) EntityColumn.MODIFIED_TIME.readResult(result);
- Long mTime1 = val.longValue();
+ long mTime1 = val.longValue();
assertEquals(mTime1, mTime);
Map<String, Object> infoColumns =
EntityColumnPrefix.INFO.readResults(result);
- assertEquals(infoMap.size(), infoColumns.size());
- for (String infoItem : infoMap.keySet()) {
- assertEquals(infoMap.get(infoItem),
- infoColumns.get(infoItem));
- }
+ assertEquals(infoMap, infoColumns);
// Remember isRelatedTo is of type Map<String, Set<String>>
for (String isRelatedToKey : isRelatedTo.keySet()) {
@@ -447,32 +454,19 @@ public class TestHBaseTimelineWriterImpl {
// Configuration
Map<String, Object> configColumns =
EntityColumnPrefix.CONFIG.readResults(result);
- assertEquals(conf.size(), configColumns.size());
- for (String configItem : conf.keySet()) {
- assertEquals(conf.get(configItem), configColumns.get(configItem));
- }
+ assertEquals(conf, configColumns);
NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
- // We got metrics back
- assertNotNull(metricMap);
- // Same number of metrics as we wrote
- assertEquals(metricValues.entrySet().size(), metricMap.entrySet()
- .size());
-
- // Iterate over original metrics and confirm that they are present
- // here.
- for (Entry<Long, Number> metricEntry : metricValues.entrySet()) {
- assertEquals(metricEntry.getValue(),
- metricMap.get(metricEntry.getKey()));
- }
+ assertEquals(metricValues, metricMap);
}
}
assertEquals(1, rowCount);
assertEquals(17, colCount);
+ // read the timeline entity using the reader this time
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
@@ -481,6 +475,30 @@ public class TestHBaseTimelineWriterImpl {
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
assertEquals(1, es1.size());
+
+ // verify attributes
+ assertEquals(id, e1.getId());
+ assertEquals(type, e1.getType());
+ assertEquals(cTime, e1.getCreatedTime());
+ assertEquals(mTime, e1.getModifiedTime());
+ Map<String, Object> infoMap2 = e1.getInfo();
+ assertEquals(infoMap, infoMap2);
+
+ Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
+ assertEquals(isRelatedTo, isRelatedTo2);
+
+ Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
+ assertEquals(relatesTo, relatesTo2);
+
+ Map<String, String> conf2 = e1.getConfigs();
+ assertEquals(conf, conf2);
+
+ Set<TimelineMetric> metrics2 = e1.getMetrics();
+ assertEquals(metrics, metrics2);
+ for (TimelineMetric metric2 : metrics2) {
+ Map<Long, Number> metricValues2 = metric2.getValues();
+ assertEquals(metricValues, metricValues2);
+ }
} finally {
if (hbi != null) {
hbi.stop();
@@ -494,9 +512,9 @@ public class TestHBaseTimelineWriterImpl {
}
private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
- String flow, Long runid, String appName, TimelineEntity te) {
+ String flow, long runid, String appName, TimelineEntity te) {
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, -1);
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
assertTrue(rowKeyComponents.length == 7);
assertEquals(user, Bytes.toString(rowKeyComponents[0]));
@@ -511,9 +529,9 @@ public class TestHBaseTimelineWriterImpl {
}
private boolean isApplicationRowKeyCorrect(byte[] rowKey, String cluster,
- String user, String flow, Long runid, String appName) {
+ String user, String flow, long runid, String appName) {
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, -1);
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
assertTrue(rowKeyComponents.length == 5);
assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
@@ -530,7 +548,7 @@ public class TestHBaseTimelineWriterImpl {
TimelineEvent event = new TimelineEvent();
String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
event.setId(eventId);
- Long expTs = 1436512802000L;
+ long expTs = 1436512802000L;
event.setTimestamp(expTs);
String expKey = "foo_event";
Object expVal = "test";
@@ -577,24 +595,25 @@ public class TestHBaseTimelineWriterImpl {
assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
appName));
- Map<String, Object> eventsResult =
- ApplicationColumnPrefix.EVENT.readResults(result);
+ Map<?, Object> eventsResult =
+ ApplicationColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
// there should be only one event
assertEquals(1, eventsResult.size());
- // key name for the event
- byte[] compoundColumnQualifierBytes =
- Separator.VALUES.join(Bytes.toBytes(eventId),
- Bytes.toBytes(TimelineWriterUtils.invert(expTs)),
- Bytes.toBytes(expKey));
- String valueKey = Bytes.toString(compoundColumnQualifierBytes);
- for (Map.Entry<String, Object> e : eventsResult.entrySet()) {
- // the value key must match
- assertEquals(valueKey, e.getKey());
+ for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
+ // the qualifier is a compound key
+ // hence match individual values
+ byte[][] karr = (byte[][])e.getKey();
+ assertEquals(3, karr.length);
+ assertEquals(eventId, Bytes.toString(karr[0]));
+ assertEquals(TimelineWriterUtils.invert(expTs), Bytes.toLong(karr[1]));
+ assertEquals(expKey, Bytes.toString(karr[2]));
Object value = e.getValue();
// there should be only one timestamp and value
assertEquals(expVal, value.toString());
}
+ // read the timeline entity using the reader this time
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
@@ -613,6 +632,21 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(1, es1.size());
assertEquals(1, es2.size());
assertEquals(es1, es2);
+
+ // check the events
+ NavigableSet<TimelineEvent> events = e1.getEvents();
+ // there should be only one event
+ assertEquals(1, events.size());
+ for (TimelineEvent e : events) {
+ assertEquals(eventId, e.getId());
+ assertEquals(expTs, e.getTimestamp());
+ Map<String,Object> info = e.getInfo();
+ assertEquals(1, info.size());
+ for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
+ assertEquals(expKey, infoEntry.getKey());
+ assertEquals(expVal, infoEntry.getValue());
+ }
+ }
} finally {
if (hbi != null) {
hbi.stop();
@@ -630,7 +664,7 @@ public class TestHBaseTimelineWriterImpl {
TimelineEvent event = new TimelineEvent();
String eventId = "foo_event_id";
event.setId(eventId);
- Long expTs = 1436512802000L;
+ long expTs = 1436512802000L;
event.setTimestamp(expTs);
final TimelineEntity entity = new TimelineEntity();
@@ -678,22 +712,21 @@ public class TestHBaseTimelineWriterImpl {
assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
entity));
- Map<String, Object> eventsResult =
- EntityColumnPrefix.EVENT.readResults(result);
+ Map<?, Object> eventsResult =
+ EntityColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
// there should be only one event
assertEquals(1, eventsResult.size());
- // key name for the event
- byte[] compoundColumnQualifierWithTsBytes =
- Separator.VALUES.join(Bytes.toBytes(eventId),
- Bytes.toBytes(TimelineWriterUtils.invert(expTs)));
- byte[] compoundColumnQualifierBytes =
- Separator.VALUES.join(compoundColumnQualifierWithTsBytes,
- null);
- String valueKey = Bytes.toString(compoundColumnQualifierBytes);
- for (Map.Entry<String, Object> e :
- eventsResult.entrySet()) {
- // the column qualifier key must match
- assertEquals(valueKey, e.getKey());
+ for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
+ // the qualifier is a compound key
+ // hence match individual values
+ byte[][] karr = (byte[][])e.getKey();
+ assertEquals(3, karr.length);
+ assertEquals(eventId, Bytes.toString(karr[0]));
+ assertEquals(TimelineWriterUtils.invert(expTs),
+ Bytes.toLong(karr[1]));
+ // key must be empty
+ assertEquals(0, karr[2].length);
Object value = e.getValue();
// value should be empty
assertEquals("", value.toString());
@@ -702,6 +735,7 @@ public class TestHBaseTimelineWriterImpl {
}
assertEquals(1, rowCount);
+ // read the timeline entity using the reader this time
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
@@ -710,6 +744,17 @@ public class TestHBaseTimelineWriterImpl {
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
assertEquals(1, es1.size());
+
+ // check the events
+ NavigableSet<TimelineEvent> events = e1.getEvents();
+ // there should be only one event
+ assertEquals(1, events.size());
+ for (TimelineEvent e : events) {
+ assertEquals(eventId, e.getId());
+ assertEquals(expTs, e.getTimestamp());
+ Map<String,Object> info = e.getInfo();
+ assertTrue(info == null || info.isEmpty());
+ }
} finally {
hbi.stop();
hbi.close();
[19/50] [abbrv] hadoop git commit: YARN-3792. Test case failures in
TestDistributedShell and some issue fixes related to ATSV2 (Naganarasimha G R
via sjlee)
Posted by sj...@apache.org.
YARN-3792. Test case failures in TestDistributedShell and some issue fixes related to ATSV2 (Naganarasimha G R via sjlee)
(cherry picked from commit 84f37f1c7eefec6d139cbf091c50d6c06f734323)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7988a71e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7988a71e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7988a71e
Branch: refs/heads/YARN-2928-rebase
Commit: 7988a71e20f51f3d2edf4a4f9ad05c30d12a9099
Parents: 3599c8e
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Jun 22 20:47:56 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:37 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 33 +++----
.../applications/distributedshell/Client.java | 2 +-
.../distributedshell/TestDistributedShell.java | 91 +++++++++++++-------
.../TestDistributedShellWithNodeLabels.java | 9 +-
.../client/api/impl/TimelineClientImpl.java | 8 ++
.../application/ApplicationImpl.java | 4 +-
.../monitor/ContainersMonitorImpl.java | 15 ++--
.../RMTimelineCollectorManager.java | 2 +-
.../collector/NodeTimelineCollectorManager.java | 14 ---
.../PerNodeTimelineCollectorsAuxService.java | 3 +-
.../collector/TimelineCollectorManager.java | 2 +-
11 files changed, 107 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0dc2058..8750056 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -35,9 +35,6 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3333. Rename TimelineAggregator etc. to TimelineCollector. (Sangjin Lee
via junping_du)
- YARN-3377. Fixed test failure in TestTimelineServiceClientIntegration.
- (Sangjin Lee via zjshen)
-
YARN-3034. Implement RM starting its timeline collector. (Naganarasimha G R
via junping_du)
@@ -61,27 +58,15 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3551. Consolidate data model change according to the backend
implementation (Zhijie Shen via sjlee)
- YARN-3562. unit tests failures and issues found from findbug from earlier
- ATS checkins (Naganarasimha G R via sjlee)
-
YARN-3134. Implemented Phoenix timeline writer to access HBase backend. (Li
Lu via zjshen)
YARN-3529. Added mini HBase cluster and Phoenix support to timeline service
v2 unit tests. (Li Lu via zjshen)
- YARN-3634. TestMRTimelineEventHandling and TestApplication are broken. (
- Sangjin Lee via junping_du)
-
YARN-3411. [Storage implementation] explore the native HBase write schema
for storage (Vrushali C via sjlee)
- YARN-3726. Fix TestHBaseTimelineWriterImpl unit test failure by fixing its
- test data (Vrushali C via sjlee)
-
- YARN-3721. build is broken on YARN-2928 branch due to possible dependency
- cycle (Li Lu via sjlee)
-
YARN-3044. Made RM write app, attempt and optional container lifecycle
events to timeline service v2. (Naganarasimha G R via zjshen)
@@ -100,6 +85,24 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
BUG FIXES
+ YARN-3377. Fixed test failure in TestTimelineServiceClientIntegration.
+ (Sangjin Lee via zjshen)
+
+ YARN-3562. unit tests failures and issues found from findbug from earlier
+ ATS checkins (Naganarasimha G R via sjlee)
+
+ YARN-3634. TestMRTimelineEventHandling and TestApplication are broken. (
+ Sangjin Lee via junping_du)
+
+ YARN-3726. Fix TestHBaseTimelineWriterImpl unit test failure by fixing its
+ test data (Vrushali C via sjlee)
+
+ YARN-3721. build is broken on YARN-2928 branch due to possible dependency
+ cycle (Li Lu via sjlee)
+
+ YARN-3792. Test case failures in TestDistributedShell and some issue fixes
+ related to ATSV2 (Naganarasimha G R via sjlee)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index ff2f594..a588544 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -463,7 +463,7 @@ public class Client {
}
if (cliParser.hasOption("flow_run_id")) {
try {
- flowRunId = Long.valueOf(cliParser.getOptionValue("flow_run_id"));
+ flowRunId = Long.parseLong(cliParser.getOptionValue("flow_run_id"));
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
"Flow run is not a valid long value", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index d31f2c8..b8a7abf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@@ -85,28 +86,33 @@ public class TestDistributedShell {
@Before
public void setup() throws Exception {
- setupInternal(NUM_NMS);
+ setupInternal(NUM_NMS, currTestName);
}
- protected void setupInternal(int numNodeManager) throws Exception {
-
+ protected void setupInternal(int numNodeManager, TestName testName)
+ throws Exception {
LOG.info("Starting up YARN cluster");
conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
conf.set("yarn.log.dir", "target");
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- // mark if we need to launch the v1 timeline server
- boolean enableATSV1 = false;
- if (!currTestName.getMethodName().toLowerCase().contains("v2")) {
+
+ if (!testName.getMethodName().toLowerCase().contains("v2")) {
// disable aux-service based timeline collectors
conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
- enableATSV1 = true;
+ conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
+ true);
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, false);
} else {
// enable aux-service based timeline collectors
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
- conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
- + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
+ conf.set(YarnConfiguration.NM_AUX_SERVICES + "."
+ + TIMELINE_AUX_SERVICE_NAME + ".class",
+ PerNodeTimelineCollectorsAuxService.class.getName());
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
+ false);
}
conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
@@ -120,12 +126,11 @@ public class TestDistributedShell {
conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
true);
- conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
if (yarnCluster == null) {
yarnCluster =
new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
- numNodeManager, 1, 1, enableATSV1);
+ numNodeManager, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
@@ -295,13 +300,15 @@ public class TestDistributedShell {
if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) {
verified = true;
}
- if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) {
+
+ if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
+ && appReport.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) {
break;
}
}
Assert.assertTrue(errorMessage, verified);
t.join();
- LOG.info("Client run completed. Result=" + result);
+ LOG.info("Client run completed for testDSShell. Result=" + result);
Assert.assertTrue(result.get());
if (!isTestingTimelineV2) {
@@ -356,9 +363,9 @@ public class TestDistributedShell {
}
}
- private void checkTimelineV2(
- boolean haveDomain, ApplicationId appId, boolean defaultFlow)
- throws Exception {
+ private void checkTimelineV2(boolean haveDomain, ApplicationId appId,
+ boolean defaultFlow) throws Exception {
+ LOG.info("Started checkTimelineV2 ");
// For PoC check in /tmp/timeline_service_data YARN-3264
String tmpRoot =
FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
@@ -409,12 +416,29 @@ public class TestDistributedShell {
verifyEntityTypeFileExists(basePath,
TimelineEntityType.YARN_APPLICATION.toString(),
appMetricsTimestampFileName);
- verifyStringExistsSpecifiedTimes(appEntityFile,
- ApplicationMetricsConstants.CREATED_EVENT_TYPE, 1,
- "Application created event should be published atleast once");
- verifyStringExistsSpecifiedTimes(appEntityFile,
- ApplicationMetricsConstants.FINISHED_EVENT_TYPE, 1,
- "Application finished event should be published atleast once");
+ Assert.assertEquals(
+ "Application created event should be published atleast once",
+ 1,
+ getNumOfStringOccurences(appEntityFile,
+ ApplicationMetricsConstants.CREATED_EVENT_TYPE));
+
+ // to avoid race condition of testcase, atleast check 4 times with sleep
+ // of 500ms
+ long numOfStringOccurences = 0;
+ for (int i = 0; i < 4; i++) {
+ numOfStringOccurences =
+ getNumOfStringOccurences(appEntityFile,
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ if (numOfStringOccurences > 0) {
+ break;
+ } else {
+ Thread.sleep(500l);
+ }
+ }
+ Assert.assertEquals(
+ "Application finished event should be published atleast once",
+ 1,
+ numOfStringOccurences);
// Verify RM posting AppAttempt life cycle Events are getting published
String appAttemptMetricsTimestampFileName =
@@ -425,12 +449,17 @@ public class TestDistributedShell {
verifyEntityTypeFileExists(basePath,
TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
appAttemptMetricsTimestampFileName);
- verifyStringExistsSpecifiedTimes(appAttemptEntityFile,
- AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE, 1,
- "AppAttempt register event should be published atleast once");
- verifyStringExistsSpecifiedTimes(appAttemptEntityFile,
- AppAttemptMetricsConstants.FINISHED_EVENT_TYPE, 1,
- "AppAttempt finished event should be published atleast once");
+ Assert.assertEquals(
+ "AppAttempt register event should be published atleast once",
+ 1,
+ getNumOfStringOccurences(appAttemptEntityFile,
+ AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE));
+
+ Assert.assertEquals(
+ "AppAttempt finished event should be published atleast once",
+ 1,
+ getNumOfStringOccurences(appAttemptEntityFile,
+ AppAttemptMetricsConstants.FINISHED_EVENT_TYPE));
} finally {
FileUtils.deleteDirectory(tmpRootFolder.getParentFile());
}
@@ -449,8 +478,7 @@ public class TestDistributedShell {
return entityFile;
}
- private void verifyStringExistsSpecifiedTimes(File entityFile,
- String searchString, long expectedNumOfTimes, String errorMsg)
+ private long getNumOfStringOccurences(File entityFile, String searchString)
throws IOException {
BufferedReader reader = null;
String strLine;
@@ -464,7 +492,7 @@ public class TestDistributedShell {
} finally {
reader.close();
}
- Assert.assertEquals(errorMsg, expectedNumOfTimes, actualCount);
+ return actualCount;
}
/**
@@ -1109,4 +1137,3 @@ public class TestDistributedShell {
return numOfWords;
}
}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java
index b62b091..496817d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java
@@ -30,7 +30,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import com.google.common.collect.ImmutableMap;
@@ -40,11 +42,14 @@ public class TestDistributedShellWithNodeLabels {
static final int NUM_NMS = 2;
TestDistributedShell distShellTest;
-
+
+ @Rule
+ public TestName currTestName = new TestName();
+
@Before
public void setup() throws Exception {
distShellTest = new TestDistributedShell();
- distShellTest.setupInternal(NUM_NMS);
+ distShellTest.setupInternal(NUM_NMS,currTestName);
}
private void initializeNodeLabels() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index d30f7b1..8312b6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -383,6 +383,14 @@ public class TimelineClientImpl extends TimelineClient {
// timelineServiceAddress could haven't be initialized yet
// or stale (only for new timeline service)
int retries = pollTimelineServiceAddress(this.maxServiceRetries);
+ if (timelineServiceAddress == null) {
+ String errMessage = "TimelineClient has reached to max retry times : "
+ + this.maxServiceRetries
+ + ", but failed to fetch timeline service address. Please verify"
+ + " Timeline Auxillary Service is configured in all the NMs";
+ LOG.error(errMessage);
+ throw new YarnException(errMessage);
+ }
// timelineServiceAddress could be stale, add retry logic here.
boolean needRetry = true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 9319b6a..6e87cfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -104,11 +104,11 @@ public class ApplicationImpl implements Application {
stateMachine = stateMachineFactory.make(this);
Configuration conf = context.getConf();
if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
- createAndStartTimelienClient(conf);
+ createAndStartTimelineClient(conf);
}
}
- private void createAndStartTimelienClient(Configuration conf) {
+ private void createAndStartTimelineClient(Configuration conf) {
// create and start timeline client
this.timelineClient = TimelineClient.createTimelineClient(appId);
timelineClient.init(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 6f08990..6e00e18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -96,11 +96,8 @@ public class ContainersMonitorImpl extends AbstractService implements
// For posting entities in new timeline service in a non-blocking way
// TODO replace with event loop in TimelineClient.
- private static ExecutorService threadPool =
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
- .build());
-
+ private static ExecutorService threadPool;
+
@Private
public static enum ContainerMetric {
CPU, MEMORY
@@ -220,6 +217,10 @@ public class ContainersMonitorImpl extends AbstractService implements
if (publishContainerMetricsToTimelineService) {
LOG.info("NodeManager has been configured to publish container " +
"metrics to Timeline Service V2.");
+ threadPool =
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
+ .build());
} else {
LOG.warn("NodeManager has not been configured to publish container " +
"metrics to Timeline Service V2.");
@@ -275,6 +276,9 @@ public class ContainersMonitorImpl extends AbstractService implements
// TODO remove threadPool after adding non-blocking call in TimelineClient
private static void shutdownAndAwaitTermination() {
+ if (threadPool == null) {
+ return;
+ }
threadPool.shutdown();
try {
if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
@@ -676,7 +680,6 @@ public class ContainersMonitorImpl extends AbstractService implements
timelineClient.putEntities(entity);
} catch (IOException|YarnException e) {
LOG.error("putEntityNonBlocking get failed: " + e);
- throw new RuntimeException(e.toString());
}
}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
index 7d1b657..116bf64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
@@ -56,7 +56,7 @@ public class RMTimelineCollectorManager extends TimelineCollectorManager {
if (parts.length != 2 || parts[1].isEmpty()) {
continue;
}
- switch (parts[0]) {
+ switch (parts[0].toUpperCase()) {
case TimelineUtils.FLOW_NAME_TAG_PREFIX:
collector.getTimelineEntityContext().setFlowName(parts[1]);
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
index 31051db..851ac30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
@@ -48,21 +48,11 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
-
-/**
- *
- * It is a singleton, and instances should be obtained via
- * {@link #getInstance()}.
- *
- */
@Private
@Unstable
public class NodeTimelineCollectorManager extends TimelineCollectorManager {
private static final Log LOG =
LogFactory.getLog(NodeTimelineCollectorManager.class);
- private static final NodeTimelineCollectorManager INSTANCE =
- new NodeTimelineCollectorManager();
-
// REST server for this collector manager
private HttpServer2 timelineRestServer;
@@ -73,10 +63,6 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
static final String COLLECTOR_MANAGER_ATTR_KEY = "collector.manager";
- static NodeTimelineCollectorManager getInstance() {
- return INSTANCE;
- }
-
@VisibleForTesting
protected NodeTimelineCollectorManager() {
super(NodeTimelineCollectorManager.class.getName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 36ff5c0..3ede97a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -56,8 +56,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
private final NodeTimelineCollectorManager collectorManager;
public PerNodeTimelineCollectorsAuxService() {
- // use the same singleton
- this(NodeTimelineCollectorManager.getInstance());
+ this(new NodeTimelineCollectorManager());
}
@VisibleForTesting PerNodeTimelineCollectorsAuxService(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7988a71e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index d54715c..23ad4f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -128,7 +128,7 @@ public abstract class TimelineCollectorManager extends AbstractService {
postRemove(appId, collector);
// stop the service to do clean up
collector.stop();
- LOG.info("the collector service for " + appId + " was removed");
+ LOG.info("The collector service for " + appId + " was removed");
}
return collector != null;
}
[04/50] [abbrv] hadoop git commit: MAPREDUCE-6335. Created MR job
based performance test driver for the timeline service v2. Contributed by
Sangjin Lee.
Posted by sj...@apache.org.
MAPREDUCE-6335. Created MR job based performance test driver for the timeline service v2. Contributed by Sangjin Lee.
(cherry picked from commit b689f5d43d3f5434a30fe52f1a7e12e1fc5c71f4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/628e9c1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/628e9c1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/628e9c1e
Branch: refs/heads/YARN-2928-rebase
Commit: 628e9c1ee9e33ba664c951d7e6b2e08eda4eb3cc
Parents: 0ffb56f
Author: Zhijie Shen <zj...@apache.org>
Authored: Tue Apr 28 19:46:01 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:39:30 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../mapred/TimelineServicePerformanceV2.java | 298 +++++++++++++++++++
.../apache/hadoop/test/MapredTestDriver.java | 3 +
3 files changed, 304 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/628e9c1e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 41bde85..b65876e 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -9,6 +9,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
MAPREDUCE-6327. Made MR AM use timeline service v2 API to write history
events and counters. (Junping Du via zjshen)
+ MAPREDUCE-6335. Created MR job based performance test driver for the
+ timeline service v2. (Sangjin Lee via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/628e9c1e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
new file mode 100644
index 0000000..de46617
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.SleepJob.SleepInputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+
+public class TimelineServicePerformanceV2 extends Configured implements Tool {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineServicePerformanceV2.class);
+
+ static final int NUM_MAPS_DEFAULT = 1;
+
+ static final int SIMPLE_ENTITY_WRITER = 1;
+ // constants for mtype = 1
+ static final String KBS_SENT = "kbs sent";
+ static final int KBS_SENT_DEFAULT = 1;
+ static final String TEST_TIMES = "testtimes";
+ static final int TEST_TIMES_DEFAULT = 100;
+ static final String TIMELINE_SERVICE_PERFORMANCE_RUN_ID =
+ "timeline.server.performance.run.id";
+
+ static int mapperType = SIMPLE_ENTITY_WRITER;
+
+ protected static int printUsage() {
+ // TODO is there a way to handle mapper-specific options more gracefully?
+ System.err.println(
+ "Usage: [-m <maps>] number of mappers (default: " + NUM_MAPS_DEFAULT +
+ ")\n" +
+ " [-mtype <mapper type in integer>] \n" +
+ " 1. simple entity write mapper\n" +
+ " [-s <(KBs)test>] number of KB per put (default: " +
+ KBS_SENT_DEFAULT + " KB)\n" +
+ " [-t] package sending iterations per mapper (default: " +
+ TEST_TIMES_DEFAULT + ")\n");
+ GenericOptionsParser.printGenericCommandUsage(System.err);
+ return -1;
+ }
+
+ /**
+ * Configure a job given argv.
+ */
+ public static boolean parseArgs(String[] args, Job job) throws IOException {
+ // set the defaults
+ Configuration conf = job.getConfiguration();
+ conf.setInt(MRJobConfig.NUM_MAPS, NUM_MAPS_DEFAULT);
+ conf.setInt(KBS_SENT, KBS_SENT_DEFAULT);
+ conf.setInt(TEST_TIMES, TEST_TIMES_DEFAULT);
+
+ for (int i = 0; i < args.length; i++) {
+ if (args.length == i + 1) {
+ System.out.println("ERROR: Required parameter missing from " + args[i]);
+ return printUsage() == 0;
+ }
+ try {
+ if ("-m".equals(args[i])) {
+ if (Integer.parseInt(args[++i]) > 0) {
+ job.getConfiguration()
+ .setInt(MRJobConfig.NUM_MAPS, (Integer.parseInt(args[i])));
+ }
+ } else if ("-mtype".equals(args[i])) {
+ mapperType = Integer.parseInt(args[++i]);
+ switch (mapperType) {
+ case SIMPLE_ENTITY_WRITER:
+ job.setMapperClass(SimpleEntityWriter.class);
+ break;
+ default:
+ job.setMapperClass(SimpleEntityWriter.class);
+ }
+ } else if ("-s".equals(args[i])) {
+ if (Integer.parseInt(args[++i]) > 0) {
+ conf.setInt(KBS_SENT, (Integer.parseInt(args[i])));
+ }
+ } else if ("-t".equals(args[i])) {
+ if (Integer.parseInt(args[++i]) > 0) {
+ conf.setInt(TEST_TIMES, (Integer.parseInt(args[i])));
+ }
+ } else {
+ System.out.println("Unexpected argument: " + args[i]);
+ return printUsage() == 0;
+ }
+ } catch (NumberFormatException except) {
+ System.out.println("ERROR: Integer expected instead of " + args[i]);
+ return printUsage() == 0;
+ } catch (Exception e) {
+ throw (IOException)new IOException().initCause(e);
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * TimelineServer Performance counters
+ */
+ static enum PerfCounters {
+ TIMELINE_SERVICE_WRITE_TIME,
+ TIMELINE_SERVICE_WRITE_COUNTER,
+ TIMELINE_SERVICE_WRITE_FAILURES,
+ TIMELINE_SERVICE_WRITE_KBS,
+ }
+
+ public int run(String[] args) throws Exception {
+
+ Job job = Job.getInstance(getConf());
+ job.setJarByClass(TimelineServicePerformanceV2.class);
+ job.setMapperClass(SimpleEntityWriter.class);
+ job.setInputFormatClass(SleepInputFormat.class);
+ job.setOutputFormatClass(NullOutputFormat.class);
+ job.setNumReduceTasks(0);
+ if (!parseArgs(args, job)) {
+ return -1;
+ }
+
+ // for mtype = 1
+ // use the current timestamp as the "run id" of the test: this will be used
+ // as simulating the cluster timestamp for apps
+ Configuration conf = job.getConfiguration();
+ conf.setLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
+ System.currentTimeMillis());
+
+ Date startTime = new Date();
+ System.out.println("Job started: " + startTime);
+ int ret = job.waitForCompletion(true) ? 0 : 1;
+ org.apache.hadoop.mapreduce.Counters counters = job.getCounters();
+ long writetime =
+ counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).getValue();
+ long writecounts =
+ counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).getValue();
+ long writesize =
+ counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).getValue();
+ double transacrate = writecounts * 1000 / (double)writetime;
+ double iorate = writesize * 1000 / (double)writetime;
+ int numMaps = Integer.parseInt(conf.get(MRJobConfig.NUM_MAPS));
+
+ System.out.println("TRANSACTION RATE (per mapper): " + transacrate +
+ " ops/s");
+ System.out.println("IO RATE (per mapper): " + iorate + " KB/s");
+
+ System.out.println("TRANSACTION RATE (total): " + transacrate*numMaps +
+ " ops/s");
+ System.out.println("IO RATE (total): " + iorate*numMaps + " KB/s");
+
+ return ret;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res =
+ ToolRunner.run(new Configuration(), new TimelineServicePerformanceV2(),
+ args);
+ System.exit(res);
+ }
+
+ /**
+ * To ensure that the compression really gets exercised, generate a
+ * random alphanumeric fixed length payload
+ */
+ static final char[] alphaNums = new char[] { 'a', 'b', 'c', 'd', 'e', 'f',
+ 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
+ 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D',
+ 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
+ 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2',
+ '3', '4', '5', '6', '7', '8', '9', '0', ' ' };
+
+ /**
+ * Adds simple entities with random string payload, events, metrics, and
+ * configuration.
+ */
+ public static class SimpleEntityWriter
+ extends org.apache.hadoop.mapreduce.Mapper<IntWritable,IntWritable,Writable,Writable> {
+ public void map(IntWritable key, IntWritable val, Context context)
+ throws IOException {
+
+ Configuration conf = context.getConfiguration();
+ // simulate the app id with the task id
+ int taskId = context.getTaskAttemptID().getTaskID().getId();
+ long timestamp = conf.getLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID, 0);
+ ApplicationId appId = ApplicationId.newInstance(timestamp, taskId);
+
+ // create the app level timeline collector
+ Configuration tlConf = new YarnConfiguration();
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId);
+ collector.init(tlConf);
+ collector.start();
+
+ try {
+ // set the context
+ // flow id: job name, flow run id: timestamp, user id
+ TimelineCollectorContext tlContext =
+ collector.getTimelineEntityContext();
+ tlContext.setFlowName(context.getJobName());
+ tlContext.setFlowRunId(timestamp);
+ tlContext.setUserId(context.getUser());
+
+ final int kbs = Integer.parseInt(conf.get(KBS_SENT));
+
+ long totalTime = 0;
+ final int testtimes = Integer.parseInt(conf.get(TEST_TIMES));
+ final Random rand = new Random();
+ final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
+ final char[] payLoad = new char[kbs * 1024];
+
+ for (int i = 0; i < testtimes; i++) {
+ // Generate a fixed length random payload
+ for (int xx = 0; xx < kbs * 1024; xx++) {
+ int alphaNumIdx = rand.nextInt(alphaNums.length);
+ payLoad[xx] = alphaNums[alphaNumIdx];
+ }
+ String entId = taskAttemptId + "_" + Integer.toString(i);
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId(entId);
+ entity.setType("FOO_ATTEMPT");
+ entity.addInfo("PERF_TEST", payLoad);
+ // add an event
+ TimelineEvent event = new TimelineEvent();
+ event.setTimestamp(System.currentTimeMillis());
+ event.addInfo("foo_event", "test");
+ entity.addEvent(event);
+ // add a metric
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId("foo_metric");
+ metric.setSingleData(123456789L);
+ entity.addMetric(metric);
+ // add a config
+ entity.addConfig("foo", "bar");
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ // use the current user for this purpose
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ long startWrite = System.nanoTime();
+ try {
+ collector.putEntities(entities, ugi);
+ } catch (Exception e) {
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
+ increment(1);
+ e.printStackTrace();
+ }
+ long endWrite = System.nanoTime();
+ totalTime += (endWrite-startWrite)/1000000L;
+ }
+ LOG.info("wrote " + testtimes + " entities (" + kbs*testtimes +
+ " kB) in " + totalTime + " ms");
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
+ increment(totalTime);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
+ increment(testtimes);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).
+ increment(kbs*testtimes);
+ } finally {
+ // clean up
+ collector.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/628e9c1e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
index 8fa82aa..5dcb143 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.mapred.TestMapRed;
import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
import org.apache.hadoop.mapred.TestTextInputFormat;
import org.apache.hadoop.mapred.ThreadedMapBenchmark;
+import org.apache.hadoop.mapred.TimelineServicePerformanceV2;
import org.apache.hadoop.mapreduce.FailJob;
import org.apache.hadoop.mapreduce.LargeSorter;
import org.apache.hadoop.mapreduce.MiniHadoopClusterManager;
@@ -90,6 +91,8 @@ public class MapredTestDriver {
pgd.addClass("fail", FailJob.class, "a job that always fails");
pgd.addClass("sleep", SleepJob.class,
"A job that sleeps at each map and reduce task.");
+ pgd.addClass("timelineperformance", TimelineServicePerformanceV2.class,
+ "A job that launch mappers to test timline service performance.");
pgd.addClass("nnbench", NNBench.class,
"A benchmark that stresses the namenode w/ MR.");
pgd.addClass("nnbenchWithoutMR", NNBenchWithoutMR.class,
[18/50] [abbrv] hadoop git commit: YARN-3051. Created storage
oriented reader interface for fetching raw entity data and made the
filesystem based implementation. Contributed by Varun Saxena.
Posted by sj...@apache.org.
YARN-3051. Created storage oriented reader interface for fetching raw entity data and made the filesystem based implementation. Contributed by Varun Saxena.
(cherry picked from commit 499ce52c7b645ec0b1cc8ac62dc9a3127b987a20)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d024123
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d024123
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d024123
Branch: refs/heads/YARN-2928-rebase
Commit: 6d0241236de0b92ccc25fc178bb6bfa00610254f
Parents: 7988a71
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Jul 6 18:11:27 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:37 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/timelineservice/TimelineEntity.java | 5 +
.../storage/FileSystemTimelineReaderImpl.java | 490 ++++++++++++++++
.../timelineservice/storage/TimelineReader.java | 162 ++++++
.../TestFileSystemTimelineReaderImpl.java | 556 +++++++++++++++++++
5 files changed, 1216 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d024123/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8750056..0d93cf8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -73,6 +73,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3801. [JDK-8] Exclude jdk.tools from hbase-client and
hbase-testing-util (Tsuyoshi Ozawa via sjlee)
+ YARN-3051. Created storage oriented reader interface for fetching raw entity
+ data and made the filesystem based implementation. (Varun Saxena via zjshen)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d024123/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index a641f32..60fba85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.util.TimelineServiceHelper;
+import org.codehaus.jackson.annotate.JsonSetter;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -335,6 +336,7 @@ public class TimelineEntity {
}
}
+ @JsonSetter("isrelatedto")
public void setIsRelatedToEntities(
Map<String, Set<String>> isRelatedToEntities) {
if (real == null) {
@@ -423,6 +425,7 @@ public class TimelineEntity {
}
}
+ @JsonSetter("relatesto")
public void setRelatesToEntities(Map<String, Set<String>> relatesToEntities) {
if (real == null) {
this.relatesToEntities =
@@ -441,6 +444,7 @@ public class TimelineEntity {
}
}
+ @JsonSetter("createdtime")
public void setCreatedTime(long createdTime) {
if (real == null) {
this.createdTime = createdTime;
@@ -458,6 +462,7 @@ public class TimelineEntity {
}
}
+ @JsonSetter("modifiedtime")
public void setModifiedTime(long modifiedTime) {
if (real == null) {
this.modifiedTime = modifiedTime;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d024123/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
new file mode 100644
index 0000000..f9f1d1d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -0,0 +1,490 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * File System based implementation for TimelineReader.
+ */
+public class FileSystemTimelineReaderImpl extends AbstractService
+ implements TimelineReader {
+
+ private static final Log LOG =
+ LogFactory.getLog(FileSystemTimelineReaderImpl.class);
+
+ private String rootPath;
+ private static final String ENTITIES_DIR = "entities";
+
+ /** Default extension for output files. */
+ private static final String TIMELINE_SERVICE_STORAGE_EXTENSION = ".thist";
+
+ @VisibleForTesting
+ /** Default extension for output files. */
+ static final String APP_FLOW_MAPPING_FILE = "app_flow_mapping.csv";
+
+ @VisibleForTesting
+ /** Config param for timeline service file system storage root. */
+ static final String TIMELINE_SERVICE_STORAGE_DIR_ROOT =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "fs-writer.root-dir";
+
+ @VisibleForTesting
+ /** Default value for storage location on local disk. */
+ static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT =
+ "/tmp/timeline_service_data";
+
+ private final CSVFormat csvFormat =
+ CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN");
+
+ public FileSystemTimelineReaderImpl() {
+ super(FileSystemTimelineReaderImpl.class.getName());
+ }
+
+ @VisibleForTesting
+ String getRootPath() {
+ return rootPath;
+ }
+
+ private static ObjectMapper mapper;
+
+ static {
+ mapper = new ObjectMapper();
+ YarnJacksonJaxbJsonProvider.configObjectMapper(mapper);
+ }
+
+ /**
+ * Deserialize a POJO object from a JSON string.
+ * @param clazz
+ * class to be desirialized
+ *
+ * @param jsonString
+ * json string to deserialize
+ * @return TimelineEntity object
+ * @throws IOException
+ * @throws JsonMappingException
+ * @throws JsonGenerationException
+ */
+ public static <T> T getTimelineRecordFromJSON(
+ String jsonString, Class<T> clazz)
+ throws JsonGenerationException, JsonMappingException, IOException {
+ return mapper.readValue(jsonString, clazz);
+ }
+
+ private static void fillFields(TimelineEntity finalEntity,
+ TimelineEntity real, EnumSet<Field> fields) {
+ if (fields.contains(Field.ALL)) {
+ finalEntity.setConfigs(real.getConfigs());
+ finalEntity.setMetrics(real.getMetrics());
+ finalEntity.setInfo(real.getInfo());
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ finalEntity.setEvents(real.getEvents());
+ return;
+ }
+ for (Field field : fields) {
+ switch(field) {
+ case CONFIGS:
+ finalEntity.setConfigs(real.getConfigs());
+ break;
+ case METRICS:
+ finalEntity.setMetrics(real.getMetrics());
+ break;
+ case INFO:
+ finalEntity.setInfo(real.getInfo());
+ break;
+ case IS_RELATED_TO:
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ break;
+ case RELATES_TO:
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ break;
+ case EVENTS:
+ finalEntity.setEvents(real.getEvents());
+ break;
+ default:
+ continue;
+ }
+ }
+ }
+
+ private static boolean matchFilter(Object infoValue, Object filterValue) {
+ return infoValue.equals(filterValue);
+ }
+
+ private static boolean matchFilters(Map<String, ? extends Object> entityInfo,
+ Map<String, ? extends Object> filters) {
+ if (entityInfo == null || entityInfo.isEmpty()) {
+ return false;
+ }
+ for (Map.Entry<String, ? extends Object> filter : filters.entrySet()) {
+ Object infoValue = entityInfo.get(filter.getKey());
+ if (infoValue == null) {
+ return false;
+ }
+ if (!matchFilter(infoValue, filter.getValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private String getFlowRunPath(String userId, String clusterId, String flowId,
+ Long flowRunId, String appId)
+ throws IOException {
+ if (userId != null && flowId != null && flowRunId != null) {
+ return userId + "/" + flowId + "/" + flowRunId;
+ }
+ if (clusterId == null || appId == null) {
+ throw new IOException("Unable to get flow info");
+ }
+ String appFlowMappingFile = rootPath + "/" + ENTITIES_DIR + "/" +
+ clusterId + "/" + APP_FLOW_MAPPING_FILE;
+ try (BufferedReader reader =
+ new BufferedReader(new InputStreamReader(
+ new FileInputStream(
+ appFlowMappingFile), Charset.forName("UTF-8")));
+ CSVParser parser = new CSVParser(reader, csvFormat)) {
+ for (CSVRecord record : parser.getRecords()) {
+ if (record.size() < 4) {
+ continue;
+ }
+ String applicationId = record.get("APP");
+ if (applicationId != null && !applicationId.trim().isEmpty() &&
+ !applicationId.trim().equals(appId)) {
+ continue;
+ }
+ return record.get(1).trim() + "/" + record.get(2).trim() + "/" +
+ record.get(3).trim();
+ }
+ parser.close();
+ }
+ throw new IOException("Unable to get flow info");
+ }
+
+ private static boolean matchMetricFilters(Set<TimelineMetric> metrics,
+ Set<String> metricFilters) {
+ Set<String> tempMetrics = new HashSet<String>();
+ for (TimelineMetric metric : metrics) {
+ tempMetrics.add(metric.getId());
+ }
+
+ for (String metricFilter : metricFilters) {
+ if (!tempMetrics.contains(metricFilter)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean matchEventFilters(Set<TimelineEvent> entityEvents,
+ Set<String> eventFilters) {
+ Set<String> tempEvents = new HashSet<String>();
+ for (TimelineEvent event : entityEvents) {
+ tempEvents.add(event.getId());
+ }
+
+ for (String eventFilter : eventFilters) {
+ if (!tempEvents.contains(eventFilter)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static TimelineEntity createEntityToBeReturned(TimelineEntity entity,
+ EnumSet<Field> fieldsToRetrieve) {
+ TimelineEntity entityToBeReturned = new TimelineEntity();
+ entityToBeReturned.setIdentifier(entity.getIdentifier());
+ entityToBeReturned.setCreatedTime(entity.getCreatedTime());
+ entityToBeReturned.setModifiedTime(entity.getModifiedTime());
+ if (fieldsToRetrieve != null) {
+ fillFields(entityToBeReturned, entity, fieldsToRetrieve);
+ }
+ return entityToBeReturned;
+ }
+
+ private static boolean isTimeInRange(Long time, Long timeBegin,
+ Long timeEnd) {
+ return (time >= timeBegin) && (time <= timeEnd);
+ }
+
+ private static boolean matchRelations(
+ Map<String, Set<String>> entityRelations,
+ Map<String, Set<String>> relations) {
+ for (Map.Entry<String, Set<String>> relation : relations.entrySet()) {
+ Set<String> ids = entityRelations.get(relation.getKey());
+ if (ids == null) {
+ return false;
+ }
+ for (String id : relation.getValue()) {
+ if (!ids.contains(id)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ private static void mergeEntities(TimelineEntity entity1,
+ TimelineEntity entity2) {
+ // Ideally created time wont change except in the case of issue from client.
+ if (entity2.getCreatedTime() > 0) {
+ entity1.setCreatedTime(entity2.getCreatedTime());
+ }
+ if (entity2.getModifiedTime() > 0) {
+ entity1.setModifiedTime(entity2.getModifiedTime());
+ }
+ for (Entry<String, String> configEntry : entity2.getConfigs().entrySet()) {
+ entity1.addConfig(configEntry.getKey(), configEntry.getValue());
+ }
+ for (Entry<String, Object> infoEntry : entity2.getInfo().entrySet()) {
+ entity1.addInfo(infoEntry.getKey(), infoEntry.getValue());
+ }
+ for (Entry<String, Set<String>> isRelatedToEntry :
+ entity2.getIsRelatedToEntities().entrySet()) {
+ String type = isRelatedToEntry.getKey();
+ for (String entityId : isRelatedToEntry.getValue()) {
+ entity1.addIsRelatedToEntity(type, entityId);
+ }
+ }
+ for (Entry<String, Set<String>> relatesToEntry :
+ entity2.getRelatesToEntities().entrySet()) {
+ String type = relatesToEntry.getKey();
+ for (String entityId : relatesToEntry.getValue()) {
+ entity1.addRelatesToEntity(type, entityId);
+ }
+ }
+ for (TimelineEvent event : entity2.getEvents()) {
+ entity1.addEvent(event);
+ }
+ for (TimelineMetric metric2 : entity2.getMetrics()) {
+ boolean found = false;
+ for (TimelineMetric metric1 : entity1.getMetrics()) {
+ if (metric1.getId().equals(metric2.getId())) {
+ metric1.addValues(metric2.getValues());
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ entity1.addMetric(metric2);
+ }
+ }
+ }
+
+ private static TimelineEntity readEntityFromFile(BufferedReader reader)
+ throws IOException {
+ TimelineEntity entity =
+ getTimelineRecordFromJSON(reader.readLine(), TimelineEntity.class);
+ String entityStr = "";
+ while ((entityStr = reader.readLine()) != null) {
+ if (entityStr.trim().isEmpty()) {
+ continue;
+ }
+ TimelineEntity anotherEntity =
+ getTimelineRecordFromJSON(entityStr, TimelineEntity.class);
+ if (!entity.getId().equals(anotherEntity.getId()) ||
+ !entity.getType().equals(anotherEntity.getType())) {
+ continue;
+ }
+ mergeEntities(entity, anotherEntity);
+ }
+ return entity;
+ }
+
+ private Set<TimelineEntity> getEntities(File dir, String entityType,
+ Long limit, Long createdTimeBegin,
+ Long createdTimeEnd, Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException {
+ if (limit == null || limit <= 0) {
+ limit = DEFAULT_LIMIT;
+ }
+ if (createdTimeBegin == null || createdTimeBegin <= 0) {
+ createdTimeBegin = 0L;
+ }
+ if (createdTimeEnd == null || createdTimeEnd <= 0) {
+ createdTimeEnd = Long.MAX_VALUE;
+ }
+ if (modifiedTimeBegin == null || modifiedTimeBegin <= 0) {
+ modifiedTimeBegin = 0L;
+ }
+ if (modifiedTimeEnd == null || modifiedTimeEnd <= 0) {
+ modifiedTimeEnd = Long.MAX_VALUE;
+ }
+
+ // First sort the selected entities based on created/start time.
+ Map<Long, Set<TimelineEntity>> sortedEntities =
+ new TreeMap<>(
+ new Comparator<Long>() {
+ @Override
+ public int compare(Long l1, Long l2) {
+ return l2.compareTo(l1);
+ }
+ }
+ );
+ for (File entityFile : dir.listFiles()) {
+ if (!entityFile.getName().contains(TIMELINE_SERVICE_STORAGE_EXTENSION)) {
+ continue;
+ }
+ try (BufferedReader reader =
+ new BufferedReader(
+ new InputStreamReader(
+ new FileInputStream(
+ entityFile), Charset.forName("UTF-8")))) {
+ TimelineEntity entity = readEntityFromFile(reader);
+ if (!entity.getType().equals(entityType)) {
+ continue;
+ }
+ if (!isTimeInRange(entity.getCreatedTime(), createdTimeBegin,
+ createdTimeEnd)) {
+ continue;
+ }
+ if (!isTimeInRange(entity.getModifiedTime(), modifiedTimeBegin,
+ modifiedTimeEnd)) {
+ continue;
+ }
+ if (relatesTo != null && !relatesTo.isEmpty() &&
+ !matchRelations(entity.getRelatesToEntities(), relatesTo)) {
+ continue;
+ }
+ if (isRelatedTo != null && !isRelatedTo.isEmpty() &&
+ !matchRelations(entity.getIsRelatedToEntities(), isRelatedTo)) {
+ continue;
+ }
+ if (infoFilters != null && !infoFilters.isEmpty() &&
+ !matchFilters(entity.getInfo(), infoFilters)) {
+ continue;
+ }
+ if (configFilters != null && !configFilters.isEmpty() &&
+ !matchFilters(entity.getConfigs(), configFilters)) {
+ continue;
+ }
+ if (metricFilters != null && !metricFilters.isEmpty() &&
+ !matchMetricFilters(entity.getMetrics(), metricFilters)) {
+ continue;
+ }
+ if (eventFilters != null && !eventFilters.isEmpty() &&
+ !matchEventFilters(entity.getEvents(), eventFilters)) {
+ continue;
+ }
+ TimelineEntity entityToBeReturned =
+ createEntityToBeReturned(entity, fieldsToRetrieve);
+ Set<TimelineEntity> entitiesCreatedAtSameTime =
+ sortedEntities.get(entityToBeReturned.getCreatedTime());
+ if (entitiesCreatedAtSameTime == null) {
+ entitiesCreatedAtSameTime = new HashSet<TimelineEntity>();
+ }
+ entitiesCreatedAtSameTime.add(entityToBeReturned);
+ sortedEntities.put(
+ entityToBeReturned.getCreatedTime(), entitiesCreatedAtSameTime);
+ }
+ }
+
+ Set<TimelineEntity> entities = new HashSet<TimelineEntity>();
+ long entitiesAdded = 0;
+ for (Set<TimelineEntity> entitySet : sortedEntities.values()) {
+ for (TimelineEntity entity : entitySet) {
+ entities.add(entity);
+ ++entitiesAdded;
+ if (entitiesAdded >= limit) {
+ return entities;
+ }
+ }
+ }
+ return entities;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ rootPath = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public TimelineEntity getEntity(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fieldsToRetrieve) throws IOException {
+ String flowRunPath = getFlowRunPath(userId, clusterId, flowId,
+ flowRunId, appId);
+ File dir = new File(new File(rootPath, ENTITIES_DIR),
+ clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType);
+ File entityFile =
+ new File(dir, entityId + TIMELINE_SERVICE_STORAGE_EXTENSION);
+ try (BufferedReader reader =
+ new BufferedReader(new InputStreamReader(
+ new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
+ TimelineEntity entity = readEntityFromFile(reader);
+ return createEntityToBeReturned(entity, fieldsToRetrieve);
+ }
+ }
+
+ @Override
+ public Set<TimelineEntity> getEntities(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException {
+ String flowRunPath =
+ getFlowRunPath(userId, clusterId, flowId, flowRunId, appId);
+ File dir =
+ new File(new File(rootPath, ENTITIES_DIR),
+ clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType);
+ return getEntities(dir, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+ eventFilters, fieldsToRetrieve);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d024123/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
new file mode 100644
index 0000000..e4e305e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+
+/** ATSv2 reader interface. */
+@Private
+@Unstable
+public interface TimelineReader extends Service {
+
+ /**
+ * Default limit for {@link #getEntities}.
+ */
+ long DEFAULT_LIMIT = 100;
+
+ /**
+ * Possible fields to retrieve for {@link #getEntities} and
+ * {@link #getEntity}.
+ */
+ public enum Field {
+ ALL,
+ EVENTS,
+ INFO,
+ METRICS,
+ CONFIGS,
+ RELATES_TO,
+ IS_RELATED_TO
+ }
+
+ /**
+ * <p>The API to fetch the single entity given the entity identifier in the
+ * scope of the given context.</p>
+ *
+ * @param userId
+ * Context user Id(optional).
+ * @param clusterId
+ * Context cluster Id(mandatory).
+ * @param flowId
+ * Context flow Id (optional).
+ * @param flowRunId
+ * Context flow run Id (optional).
+ * @param appId
+ * Context app Id (mandatory)
+ * @param entityType
+ * Entity type (mandatory)
+ * @param entityId
+ * Entity Id (mandatory)
+ * @param fieldsToRetrieve
+ * Specifies which fields of the entity object to retrieve(optional), see
+ * {@link Field}. If null, retrieves 4 fields namely entity id,
+ * entity type, entity created time and entity modified time. All
+ * entities will be returned if {@link Field#ALL} is specified.
+ * @return a {@link TimelineEntity} instance or null. The entity will
+ * contain the metadata plus the given fields to retrieve.
+ * @throws IOException
+ */
+ TimelineEntity getEntity(String userId, String clusterId, String flowId,
+ Long flowRunId, String appId, String entityType, String entityId,
+ EnumSet<Field> fieldsToRetrieve) throws IOException;
+
+ /**
+ * <p>The API to search for a set of entities of the given the entity type in
+ * the scope of the given context which matches the given predicates. The
+ * predicates include the created/modified time window, limit to number of
+ * entities to be returned, and the entities can be filtered by checking
+ * whether they contain the given info/configs entries in the form of
+ * key/value pairs, given metrics in the form of metricsIds and its relation
+ * with metric values given events in the form of the Ids, and whether they
+ * relate to/are related to other entities. For those parameters which have
+ * multiple entries, the qualified entity needs to meet all or them.</p>
+ *
+ * @param userId
+ * Context user Id(optional).
+ * @param clusterId
+ * Context cluster Id(mandatory).
+ * @param flowId
+ * Context flow Id (optional).
+ * @param flowRunId
+ * Context flow run Id (optional).
+ * @param appId
+ * Context app Id (mandatory)
+ * @param entityType
+ * Entity type (mandatory)
+ * @param limit
+ * A limit on the number of entities to return (optional). If null or <=0,
+ * defaults to {@link #DEFAULT_LIMIT}.
+ * @param createdTimeBegin
+ * Matched entities should not be created before this timestamp (optional).
+ * If null or <=0, defaults to 0.
+ * @param createdTimeEnd
+ * Matched entities should not be created after this timestamp (optional).
+ * If null or <=0, defaults to {@link Long#MAX_VALUE}.
+ * @param modifiedTimeBegin
+ * Matched entities should not be modified before this timestamp
+ * (optional). If null or <=0, defaults to 0.
+ * @param modifiedTimeEnd
+ * Matched entities should not be modified after this timestamp (optional).
+ * If null or <=0, defaults to {@link Long#MAX_VALUE}.
+ * @param relatesTo
+ * Matched entities should relate to given entities (optional).
+ * @param isRelatedTo
+ * Matched entities should be related to given entities (optional).
+ * @param infoFilters
+ * Matched entities should have exact matches to the given info represented
+ * as key-value pairs (optional). If null or empty, the filter is not
+ * applied.
+ * @param configFilters
+ * Matched entities should have exact matches to the given configs
+ * represented as key-value pairs (optional). If null or empty, the filter
+ * is not applied.
+ * @param metricFilters
+ * Matched entities should contain the given metrics (optional). If null
+ * or empty, the filter is not applied.
+ * @param eventFilters
+ * Matched entities should contain the given events (optional). If null
+ * or empty, the filter is not applied.
+ * @param fieldsToRetrieve
+ * Specifies which fields of the entity object to retrieve(optional), see
+ * {@link Field}. If null, retrieves 4 fields namely entity id,
+ * entity type, entity created time and entity modified time. All
+ * entities will be returned if {@link Field#ALL} is specified.
+ * @return A set of {@link TimelineEntity} instances of the given entity type
+ * in the given context scope which matches the given predicates
+ * ordered by created time, descending. Each entity will only contain the
+ * metadata(id, type, created and modified times) plus the given fields to
+ * retrieve.
+ * @throws IOException
+ */
+ Set<TimelineEntity> getEntities(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d024123/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
new file mode 100644
index 0000000..4e23e49
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
@@ -0,0 +1,556 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestFileSystemTimelineReaderImpl {
+
+ private static final String rootDir =
+ FileSystemTimelineReaderImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT;
+ FileSystemTimelineReaderImpl reader;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ loadEntityData();
+ // Create app flow mapping file.
+ CSVFormat format =
+ CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN");
+ String appFlowMappingFile = rootDir + "/entities/cluster1/" +
+ FileSystemTimelineReaderImpl.APP_FLOW_MAPPING_FILE;
+ try (PrintWriter out =
+ new PrintWriter(new BufferedWriter(
+ new FileWriter(appFlowMappingFile, true)));
+ CSVPrinter printer = new CSVPrinter(out, format)){
+ printer.printRecord("app1", "user1", "flow1", 1);
+ printer.printRecord("app2","user1","flow1,flow",1);
+ printer.close();
+ }
+ (new File(rootDir)).deleteOnExit();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ FileUtils.deleteDirectory(new File(rootDir));
+ }
+
+ @Before
+ public void init() throws Exception {
+ reader = new FileSystemTimelineReaderImpl();
+ Configuration conf = new YarnConfiguration();
+ conf.set(FileSystemTimelineReaderImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ rootDir);
+ reader.init(conf);
+ }
+
+ private static void writeEntityFile(TimelineEntity entity, File dir)
+ throws Exception {
+ if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ throw new IOException("Could not create directories for " + dir);
+ }
+ }
+ String fileName = dir.getAbsolutePath() + "/" + entity.getId() + ".thist";
+ try (PrintWriter out =
+ new PrintWriter(new BufferedWriter(new FileWriter(fileName, true)))){
+ out.println(TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ out.write("\n");
+ out.close();
+ }
+ }
+
+ private static void loadEntityData() throws Exception {
+ File appDir = new File(rootDir +
+ "/entities/cluster1/user1/flow1/1/app1/app/");
+ TimelineEntity entity11 = new TimelineEntity();
+ entity11.setId("id_1");
+ entity11.setType("app");
+ entity11.setCreatedTime(1425016502000L);
+ entity11.setModifiedTime(1425016502050L);
+ Map<String, Object> info1 = new HashMap<String, Object>();
+ info1.put("info1", "val1");
+ entity11.addInfo(info1);
+ TimelineEvent event = new TimelineEvent();
+ event.setId("event_1");
+ event.setTimestamp(1425016502003L);
+ entity11.addEvent(event);
+ Set<TimelineMetric> metrics = new HashSet<TimelineMetric>();
+ TimelineMetric metric1 = new TimelineMetric();
+ metric1.setId("metric1");
+ metric1.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric1.addValue(1425016502006L, 113.2F);
+ metrics.add(metric1);
+ TimelineMetric metric2 = new TimelineMetric();
+ metric2.setId("metric2");
+ metric2.setType(TimelineMetric.Type.TIME_SERIES);
+ metric2.addValue(1425016502016L, 34);
+ metrics.add(metric2);
+ entity11.setMetrics(metrics);
+ Map<String,String> configs = new HashMap<String, String>();
+ configs.put("config_1", "123");
+ entity11.setConfigs(configs);
+ entity11.addRelatesToEntity("flow", "flow1");
+ entity11.addIsRelatedToEntity("type1", "tid1_1");
+ writeEntityFile(entity11, appDir);
+ TimelineEntity entity12 = new TimelineEntity();
+ entity12.setId("id_1");
+ entity12.setType("app");
+ entity12.setModifiedTime(1425016503000L);
+ configs.clear();
+ configs.put("config_2", "23");
+ configs.put("config_3", "abc");
+ entity12.addConfigs(configs);
+ metrics.clear();
+ TimelineMetric metric12 = new TimelineMetric();
+ metric12.setId("metric2");
+ metric12.setType(TimelineMetric.Type.TIME_SERIES);
+ metric12.addValue(1425016502032L, 48);
+ metric12.addValue(1425016502054L, 51);
+ metrics.add(metric12);
+ TimelineMetric metric3 = new TimelineMetric();
+ metric3.setId("metric3");
+ metric3.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric3.addValue(1425016502060L, 23L);
+ metrics.add(metric3);
+ entity12.setMetrics(metrics);
+ entity12.addIsRelatedToEntity("type1", "tid1_2");
+ entity12.addIsRelatedToEntity("type2", "tid2_1`");
+ TimelineEvent event15 = new TimelineEvent();
+ event15.setId("event_5");
+ event15.setTimestamp(1425016502017L);
+ entity12.addEvent(event15);
+ writeEntityFile(entity12, appDir);
+
+ TimelineEntity entity2 = new TimelineEntity();
+ entity2.setId("id_2");
+ entity2.setType("app");
+ entity2.setCreatedTime(1425016501050L);
+ entity2.setModifiedTime(1425016502010L);
+ Map<String, Object> info2 = new HashMap<String, Object>();
+ info1.put("info2", 4);
+ entity2.addInfo(info2);
+ Map<String,String> configs2 = new HashMap<String, String>();
+ configs2.put("config_1", "123");
+ configs2.put("config_3", "def");
+ entity2.setConfigs(configs2);
+ TimelineEvent event2 = new TimelineEvent();
+ event2.setId("event_2");
+ event2.setTimestamp(1425016501003L);
+ entity2.addEvent(event2);
+ Set<TimelineMetric> metrics2 = new HashSet<TimelineMetric>();
+ TimelineMetric metric21 = new TimelineMetric();
+ metric21.setId("metric1");
+ metric21.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric21.addValue(1425016501006L, 123.2F);
+ metrics2.add(metric21);
+ TimelineMetric metric22 = new TimelineMetric();
+ metric22.setId("metric2");
+ metric22.setType(TimelineMetric.Type.TIME_SERIES);
+ metric22.addValue(1425016501056L, 31);
+ metric22.addValue(1425016501084L, 70);
+ metrics2.add(metric22);
+ TimelineMetric metric23 = new TimelineMetric();
+ metric23.setId("metric3");
+ metric23.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric23.addValue(1425016502060L, 23L);
+ metrics2.add(metric23);
+ entity2.setMetrics(metrics2);
+ entity2.addRelatesToEntity("flow", "flow2");
+ writeEntityFile(entity2, appDir);
+
+ TimelineEntity entity3 = new TimelineEntity();
+ entity3.setId("id_3");
+ entity3.setType("app");
+ entity3.setCreatedTime(1425016501050L);
+ entity3.setModifiedTime(1425016502010L);
+ Map<String, Object> info3 = new HashMap<String, Object>();
+ info3.put("info2", 3.5);
+ entity3.addInfo(info3);
+ Map<String,String> configs3 = new HashMap<String, String>();
+ configs3.put("config_1", "123");
+ configs3.put("config_3", "abc");
+ entity3.setConfigs(configs3);
+ TimelineEvent event3 = new TimelineEvent();
+ event3.setId("event_2");
+ event3.setTimestamp(1425016501003L);
+ entity3.addEvent(event3);
+ TimelineEvent event4 = new TimelineEvent();
+ event4.setId("event_4");
+ event4.setTimestamp(1425016502006L);
+ entity3.addEvent(event4);
+ Set<TimelineMetric> metrics3 = new HashSet<TimelineMetric>();
+ TimelineMetric metric31 = new TimelineMetric();
+ metric31.setId("metric1");
+ metric31.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric31.addValue(1425016501006L, 124.8F);
+ metrics3.add(metric31);
+ TimelineMetric metric32 = new TimelineMetric();
+ metric32.setId("metric2");
+ metric32.setType(TimelineMetric.Type.TIME_SERIES);
+ metric32.addValue(1425016501056L, 31);
+ metric32.addValue(1425016501084L, 74);
+ metrics3.add(metric32);
+ entity3.setMetrics(metrics3);
+ entity3.addIsRelatedToEntity("type1", "tid1_2");
+ writeEntityFile(entity3, appDir);
+
+ TimelineEntity entity4 = new TimelineEntity();
+ entity4.setId("id_4");
+ entity4.setType("app");
+ entity4.setCreatedTime(1425016502050L);
+ entity4.setModifiedTime(1425016503010L);
+ TimelineEvent event44 = new TimelineEvent();
+ event44.setId("event_4");
+ event44.setTimestamp(1425016502003L);
+ entity4.addEvent(event44);
+ writeEntityFile(entity4, appDir);
+
+ File appDir2 = new File(rootDir +
+ "/entities/cluster1/user1/flow1,flow/1/app2/app/");
+ TimelineEntity entity5 = new TimelineEntity();
+ entity5.setId("id_5");
+ entity5.setType("app");
+ entity5.setCreatedTime(1425016502050L);
+ entity5.setModifiedTime(1425016503010L);
+ writeEntityFile(entity5, appDir2);
+ }
+
+ public TimelineReader getTimelineReader() {
+ return reader;
+ }
+
+ @Test
+ public void testGetEntityDefaultView() throws Exception {
+ // If no fields are specified, entity is returned with default view i.e.
+ // only the id, created and modified time
+ TimelineEntity result =
+ reader.getEntity("user1", "cluster1", "flow1", 1L, "app1",
+ "app", "id_1", null);
+ Assert.assertEquals(
+ (new TimelineEntity.Identifier("app", "id_1")).toString(),
+ result.getIdentifier().toString());
+ Assert.assertEquals(1425016502000L, result.getCreatedTime());
+ Assert.assertEquals(1425016503000L, result.getModifiedTime());
+ Assert.assertEquals(0, result.getConfigs().size());
+ Assert.assertEquals(0, result.getMetrics().size());
+ }
+
+ @Test
+ public void testGetEntityByClusterAndApp() throws Exception {
+ // Cluster and AppId should be enough to get an entity.
+ TimelineEntity result =
+ reader.getEntity(null, "cluster1", null, null, "app1",
+ "app", "id_1", null);
+ Assert.assertEquals(
+ (new TimelineEntity.Identifier("app", "id_1")).toString(),
+ result.getIdentifier().toString());
+ Assert.assertEquals(1425016502000L, result.getCreatedTime());
+ Assert.assertEquals(1425016503000L, result.getModifiedTime());
+ Assert.assertEquals(0, result.getConfigs().size());
+ Assert.assertEquals(0, result.getMetrics().size());
+ }
+
+ /** This test checks whether we can handle commas in app flow mapping csv */
+ @Test
+ public void testAppFlowMappingCsv() throws Exception {
+ // Test getting an entity by cluster and app where flow entry
+ // in app flow mapping csv has commas.
+ TimelineEntity result =
+ reader.getEntity(null, "cluster1", null, null, "app2",
+ "app", "id_5", null);
+ Assert.assertEquals(
+ (new TimelineEntity.Identifier("app", "id_5")).toString(),
+ result.getIdentifier().toString());
+ Assert.assertEquals(1425016502050L, result.getCreatedTime());
+ Assert.assertEquals(1425016503010L, result.getModifiedTime());
+ }
+
+ @Test
+ public void testGetEntityCustomFields() throws Exception {
+ // Specified fields in addition to default view will be returned.
+ TimelineEntity result =
+ reader.getEntity("user1", "cluster1", "flow1", 1L,
+ "app1", "app", "id_1",
+ EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS));
+ Assert.assertEquals(
+ (new TimelineEntity.Identifier("app", "id_1")).toString(),
+ result.getIdentifier().toString());
+ Assert.assertEquals(1425016502000L, result.getCreatedTime());
+ Assert.assertEquals(1425016503000L, result.getModifiedTime());
+ Assert.assertEquals(3, result.getConfigs().size());
+ Assert.assertEquals(3, result.getMetrics().size());
+ Assert.assertEquals(1, result.getInfo().size());
+ // No events will be returned
+ Assert.assertEquals(0, result.getEvents().size());
+ }
+
+ @Test
+ public void testGetEntityAllFields() throws Exception {
+ // All fields of TimelineEntity will be returned.
+ TimelineEntity result =
+ reader.getEntity("user1", "cluster1", "flow1", 1L,
+ "app1", "app", "id_1", EnumSet.of(Field.ALL));
+ Assert.assertEquals(
+ (new TimelineEntity.Identifier("app", "id_1")).toString(),
+ result.getIdentifier().toString());
+ Assert.assertEquals(1425016502000L, result.getCreatedTime());
+ Assert.assertEquals(1425016503000L, result.getModifiedTime());
+ Assert.assertEquals(3, result.getConfigs().size());
+ Assert.assertEquals(3, result.getMetrics().size());
+ // All fields including events will be returned.
+ Assert.assertEquals(2, result.getEvents().size());
+ }
+
+ @Test
+ public void testGetAllEntities() throws Exception {
+ Set<TimelineEntity> result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, null, null, null, null,
+ null, null);
+ // All 3 entities will be returned
+ Assert.assertEquals(4, result.size());
+ }
+
+ @Test
+ public void testGetEntitiesWithLimit() throws Exception {
+ Set<TimelineEntity> result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ 2L, null, null, null, null, null, null, null, null, null,
+ null, null);
+ Assert.assertEquals(2, result.size());
+ // Needs to be rewritten once hashcode and equals for
+ // TimelineEntity is implemented
+ // Entities with id_1 and id_4 should be returned,
+ // based on created time, descending.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_4")) {
+ Assert.fail("Entity not sorted by created time");
+ }
+ }
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ 3L, null, null, null, null, null, null, null, null, null,
+ null, null);
+ // Even though 2 entities out of 4 have same created time, one entity
+ // is left out due to limit
+ Assert.assertEquals(3, result.size());
+ }
+
+ @Test
+ public void testGetEntitiesByTimeWindows() throws Exception {
+ // Get entities based on created time start and end time range.
+ Set<TimelineEntity> result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, 1425016502030L, 1425016502060L, null, null, null, null, null,
+ null, null, null, null);
+ Assert.assertEquals(1, result.size());
+ // Only one entity with ID id_4 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on created time range");
+ }
+ }
+
+ // Get entities if only created time end is specified.
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, 1425016502010L, null, null, null, null, null, null,
+ null, null, null);
+ Assert.assertEquals(3, result.size());
+ for (TimelineEntity entity : result) {
+ if (entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on created time range");
+ }
+ }
+
+ // Get entities if only created time start is specified.
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, 1425016502010L, null, null, null, null, null, null, null,
+ null, null, null);
+ Assert.assertEquals(1, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on created time range");
+ }
+ }
+
+ // Get entities based on modified time start and end time range.
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, 1425016502090L, 1425016503020L, null, null, null,
+ null, null, null, null);
+ Assert.assertEquals(2, result.size());
+ // Two entities with IDs' id_1 and id_4 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on modified time range");
+ }
+ }
+
+ // Get entities if only modified time end is specified.
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, 1425016502090L, null, null, null, null,
+ null, null, null);
+ Assert.assertEquals(2, result.size());
+ for (TimelineEntity entity : result) {
+ if (entity.getId().equals("id_1") || entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on modified time range");
+ }
+ }
+
+ // Get entities if only modified time start is specified.
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, 1425016503005L, null, null, null, null, null,
+ null, null, null);
+ Assert.assertEquals(1, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on modified time range");
+ }
+ }
+ }
+
+ @Test
+ public void testGetFilteredEntities() throws Exception {
+ // Get entities based on info filters.
+ Map<String, Object> infoFilters = new HashMap<String, Object>();
+ infoFilters.put("info2", 3.5);
+ Set<TimelineEntity> result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, null, infoFilters, null, null,
+ null, null);
+ Assert.assertEquals(1, result.size());
+ // Only one entity with ID id_3 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_3")) {
+ Assert.fail("Incorrect filtering based on info filters");
+ }
+ }
+
+ // Get entities based on config filters.
+ Map<String, String> configFilters = new HashMap<String, String>();
+ configFilters.put("config_1", "123");
+ configFilters.put("config_3", "abc");
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, null, null, configFilters, null,
+ null, null);
+ Assert.assertEquals(2, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
+ Assert.fail("Incorrect filtering based on config filters");
+ }
+ }
+
+ // Get entities based on event filters.
+ Set<String> eventFilters = new HashSet<String>();
+ eventFilters.add("event_2");
+ eventFilters.add("event_4");
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, null, null, null, null,
+ eventFilters, null);
+ Assert.assertEquals(1, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_3")) {
+ Assert.fail("Incorrect filtering based on event filters");
+ }
+ }
+
+ // Get entities based on metric filters.
+ Set<String> metricFilters = new HashSet<String>();
+ metricFilters.add("metric3");
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, null, null, null, metricFilters,
+ null, null);
+ Assert.assertEquals(2, result.size());
+ // Two entities with IDs' id_1 and id_2 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
+ Assert.fail("Incorrect filtering based on metric filters");
+ }
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByRelations() throws Exception {
+ // Get entities based on relatesTo.
+ Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
+ Set<String> relatesToIds = new HashSet<String>();
+ relatesToIds.add("flow1");
+ relatesTo.put("flow", relatesToIds);
+ Set<TimelineEntity> result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, relatesTo, null, null, null, null,
+ null, null);
+ Assert.assertEquals(1, result.size());
+ // Only one entity with ID id_1 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1")) {
+ Assert.fail("Incorrect filtering based on relatesTo");
+ }
+ }
+
+ // Get entities based on isRelatedTo.
+ Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
+ Set<String> isRelatedToIds = new HashSet<String>();
+ isRelatedToIds.add("tid1_2");
+ isRelatedTo.put("type1", isRelatedToIds);
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, isRelatedTo, null, null, null,
+ null, null);
+ Assert.assertEquals(2, result.size());
+ // Two entities with IDs' id_1 and id_3 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
+ Assert.fail("Incorrect filtering based on isRelatedTo");
+ }
+ }
+ }
+}
[31/50] [abbrv] hadoop git commit: MAPREDUCE-6370. Made the timeline
service v2 test driver write event ID. Contributed by Li Lu.
Posted by sj...@apache.org.
MAPREDUCE-6370. Made the timeline service v2 test driver write event ID. Contributed by Li Lu.
(cherry picked from commit 827633ee9fee26e3e15343cbccb0b7905ae02170)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa927d87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa927d87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa927d87
Branch: refs/heads/YARN-2928-rebase
Commit: aa927d871fae20fa1e812e678226339fa8df39bf
Parents: 8349b90
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri May 22 00:00:05 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:42 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java | 1 +
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa927d87/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 17d1c9f..24e4172 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -21,6 +21,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
BUG FIXES
+ MAPREDUCE-6370. Made the timeline service v2 test driver write event ID.
+ (Li Lu via zjshen)
+
Trunk (Unreleased)
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa927d87/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
index 4ef0a14..625c32a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
@@ -97,6 +97,7 @@ class SimpleEntityWriter extends EntityWriter {
entity.addInfo("PERF_TEST", payLoad);
// add an event
TimelineEvent event = new TimelineEvent();
+ event.setId("foo_event_id");
event.setTimestamp(System.currentTimeMillis());
event.addInfo("foo_event", "test");
entity.addEvent(event);
[15/50] [abbrv] hadoop git commit: YARN-3706. Generalize native HBase
writer for additional tables (Joep Rottinghuis via sjlee)
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
new file mode 100644
index 0000000..ee57890
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Used to separate row qualifiers, column qualifiers and compount fields.
+ */
+public enum Separator {
+
+ /**
+ * separator in key or column qualifier fields
+ */
+ QUALIFIERS("!", "%0$"),
+
+ /**
+ * separator in values, and/or compound key/column qualifier fields.
+ */
+ VALUES("?", "%1$"),
+
+ /**
+ * separator in values, often used to avoid having these in qualifiers and
+ * names. Note that if we use HTML form encoding through URLEncoder, we end up
+ * getting a + for a space, which may already occur in strings, so we don't
+ * want that.
+ */
+ SPACE(" ", "%2$");
+
+ /**
+ * The string value of this separator.
+ */
+ private final String value;
+
+ /**
+ * The URLEncoded version of this separator
+ */
+ private final String encodedValue;
+
+ /**
+ * The bye representation of value.
+ */
+ private final byte[] bytes;
+
+ /**
+ * The value quoted so that it can be used as a safe regex
+ */
+ private final String quotedValue;
+
+ private static final byte[] EMPTY_BYTES = new byte[0];
+
+ /**
+ * @param value of the separator to use. Cannot be null or empty string.
+ * @param encodedValue choose something that isn't likely to occur in the data
+ * itself. Cannot be null or empty string.
+ */
+ private Separator(String value, String encodedValue) {
+ this.value = value;
+ this.encodedValue = encodedValue;
+
+ // validation
+ if (value == null || value.length() == 0 || encodedValue == null
+ || encodedValue.length() == 0) {
+ throw new IllegalArgumentException(
+ "Cannot create separator from null or empty string.");
+ }
+
+ this.bytes = Bytes.toBytes(value);
+ this.quotedValue = Pattern.quote(value);
+ }
+
+ /**
+ * Used to make token safe to be used with this separator without collisions.
+ *
+ * @param token
+ * @return the token with any occurrences of this separator URLEncoded.
+ */
+ public String encode(String token) {
+ if (token == null || token.length() == 0) {
+ // Nothing to replace
+ return token;
+ }
+ return token.replace(value, encodedValue);
+ }
+
+ /**
+ * @param token
+ * @return the token with any occurrences of the encoded separator replaced by
+ * the separator itself.
+ */
+ public String decode(String token) {
+ if (token == null || token.length() == 0) {
+ // Nothing to replace
+ return token;
+ }
+ return token.replace(encodedValue, value);
+ }
+
+ /**
+ * Encode the given separators in the token with their encoding equivalent.
+ * This means that when encoding is already present in the token itself, this
+ * is not a reversible process. See also {@link #decode(String, Separator...)}
+ *
+ * @param token containing possible separators that need to be encoded.
+ * @param separators to be encoded in the token with their URLEncoding
+ * equivalent.
+ * @return non-null byte representation of the token with occurrences of the
+ * separators encoded.
+ */
+ public static byte[] encode(String token, Separator... separators) {
+ if (token == null) {
+ return EMPTY_BYTES;
+ }
+ String result = token;
+ for (Separator separator : separators) {
+ if (separator != null) {
+ result = separator.encode(result);
+ }
+ }
+ return Bytes.toBytes(result);
+ }
+
+ /**
+ * Decode the given separators in the token with their decoding equivalent.
+ * This means that when encoding is already present in the token itself, this
+ * is not a reversible process.
+ *
+ * @param token containing possible separators that need to be encoded.
+ * @param separators to be encoded in the token with their URLEncoding
+ * equivalent.
+ * @return String representation of the token with occurrences of the URL
+ * encoded separators decoded.
+ */
+ public static String decode(byte[] token, Separator... separators) {
+ if (token == null) {
+ return null;
+ }
+ return decode(Bytes.toString(token), separators);
+ }
+
+ /**
+ * Decode the given separators in the token with their decoding equivalent.
+ * This means that when encoding is already present in the token itself, this
+ * is not a reversible process.
+ *
+ * @param token containing possible separators that need to be encoded.
+ * @param separators to be encoded in the token with their URLEncoding
+ * equivalent.
+ * @return String representation of the token with occurrences of the URL
+ * encoded separators decoded.
+ */
+ public static String decode(String token, Separator... separators) {
+ if (token == null) {
+ return null;
+ }
+ String result = token;
+ for (Separator separator : separators) {
+ if (separator != null) {
+ result = separator.decode(result);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns a single byte array containing all of the individual arrays
+ * components separated by this separator.
+ *
+ * @param components
+ * @return byte array after joining the components
+ */
+ public byte[] join(byte[]... components) {
+ if (components == null || components.length == 0) {
+ return EMPTY_BYTES;
+ }
+
+ int finalSize = 0;
+ finalSize = this.value.length() * (components.length - 1);
+ for (byte[] comp : components) {
+ if (comp != null) {
+ finalSize += comp.length;
+ }
+ }
+
+ byte[] buf = new byte[finalSize];
+ int offset = 0;
+ for (int i = 0; i < components.length; i++) {
+ if (components[i] != null) {
+ System.arraycopy(components[i], 0, buf, offset, components[i].length);
+ offset += components[i].length;
+ }
+ if (i < (components.length - 1)) {
+ System.arraycopy(this.bytes, 0, buf, offset, this.value.length());
+ offset += this.value.length();
+ }
+
+ }
+ return buf;
+ }
+
+ /**
+ * Concatenates items (as String), using this separator.
+ *
+ * @param items Items join, {@code toString()} will be called in each item.
+ * Any occurrence of the separator in the individual strings will be
+ * first encoded. Cannot be null.
+ * @return non-null joined result. Note that when separator is {@literal null}
+ * the result is simply all items concatenated and the process is not
+ * reversible through {@link #splitEncoded(String)}
+ */
+ public String joinEncoded(String... items) {
+ if (items == null || items.length == 0) {
+ return "";
+ }
+
+ StringBuilder sb = new StringBuilder(encode(items[0].toString()));
+ // Start at 1, we've already grabbed the first value at index 0
+ for (int i = 1; i < items.length; i++) {
+ sb.append(this.value);
+ sb.append(encode(items[i].toString()));
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Concatenates items (as String), using this separator.
+ *
+ * @param items Items join, {@code toString()} will be called in each item.
+ * Any occurrence of the separator in the individual strings will be
+ * first encoded. Cannot be null.
+ * @return non-null joined result. Note that when separator is {@literal null}
+ * the result is simply all items concatenated and the process is not
+ * reversible through {@link #splitEncoded(String)}
+ */
+ public String joinEncoded(Iterable<?> items) {
+ if (items == null) {
+ return "";
+ }
+ Iterator<?> i = items.iterator();
+ if (!i.hasNext()) {
+ return "";
+ }
+
+ StringBuilder sb = new StringBuilder(encode(i.next().toString()));
+ while (i.hasNext()) {
+ sb.append(this.value);
+ sb.append(encode(i.next().toString()));
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * @param compoundValue containing individual values separated by this
+ * separator, which have that separator encoded.
+ * @return non-null set of values from the compoundValue with the separator
+ * decoded.
+ */
+ public Collection<String> splitEncoded(String compoundValue) {
+ List<String> result = new ArrayList<String>();
+ if (compoundValue != null) {
+ for (String value : compoundValue.split(quotedValue)) {
+ result.add(decode(value));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Splits the source array into multiple array segments using this separator,
+ * up to a maximum of count items. This will naturally produce copied byte
+ * arrays for each of the split segments.
+ * @param source to be split
+ * @param limit on how many segments are supposed to be returned. Negative
+ * value indicates no limit on number of segments.
+ * @return source split by this separator.
+ */
+ public byte[][] split(byte[] source, int limit) {
+ return TimelineWriterUtils.split(source, this.bytes, limit);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
new file mode 100644
index 0000000..5518a27
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * contains the constants used in the context of schema accesses for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * information
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class TimelineEntitySchemaConstants {
+
+ /**
+ * Used to create a pre-split for tables starting with a username in the
+ * prefix. TODO: this may have to become a config variable (string with
+ * separators) so that different installations can presplit based on their own
+ * commonly occurring names.
+ */
+ private final static byte[][] USERNAME_SPLITS = { Bytes.toBytes("a"),
+ Bytes.toBytes("ad"), Bytes.toBytes("an"), Bytes.toBytes("b"),
+ Bytes.toBytes("ca"), Bytes.toBytes("cl"), Bytes.toBytes("d"),
+ Bytes.toBytes("e"), Bytes.toBytes("f"), Bytes.toBytes("g"),
+ Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j"),
+ Bytes.toBytes("k"), Bytes.toBytes("l"), Bytes.toBytes("m"),
+ Bytes.toBytes("n"), Bytes.toBytes("o"), Bytes.toBytes("q"),
+ Bytes.toBytes("r"), Bytes.toBytes("s"), Bytes.toBytes("se"),
+ Bytes.toBytes("t"), Bytes.toBytes("u"), Bytes.toBytes("v"),
+ Bytes.toBytes("w"), Bytes.toBytes("x"), Bytes.toBytes("y"),
+ Bytes.toBytes("z") };
+
+ /**
+ * The length at which keys auto-split
+ */
+ public static final String USERNAME_SPLIT_KEY_PREFIX_LENGTH = "4";
+
+ /**
+ * @return splits for splits where a user is a prefix.
+ */
+ public final static byte[][] getUsernameSplits() {
+ byte[][] kloon = USERNAME_SPLITS.clone();
+ // Deep copy.
+ for (int row = 0; row < USERNAME_SPLITS.length; row++) {
+ kloon[row] = Bytes.copy(USERNAME_SPLITS[row]);
+ }
+ return kloon;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
new file mode 100644
index 0000000..28a0b6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * bunch of utility functions used across TimelineWriter classes
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class TimelineWriterUtils {
+
+ /** empty bytes */
+ public static final byte[] EMPTY_BYTES = new byte[0];
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments. To identify the split
+ * ranges without the array copies, see
+ * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
+ *
+ * @param source
+ * @param separator
+ * @return byte[] array after splitting the source
+ */
+ public static byte[][] split(byte[] source, byte[] separator) {
+ return split(source, separator, -1);
+ }
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments. To identify the split
+ * ranges without the array copies, see
+ * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
+ *
+ * @param source
+ * @param separator
+ * @param limit a negative value indicates no limit on number of segments.
+ * @return byte[][] after splitting the input source
+ */
+ public static byte[][] split(byte[] source, byte[] separator, int limit) {
+ List<Range> segments = splitRanges(source, separator, limit);
+
+ byte[][] splits = new byte[segments.size()][];
+ for (int i = 0; i < segments.size(); i++) {
+ Range r = segments.get(i);
+ byte[] tmp = new byte[r.length()];
+ if (tmp.length > 0) {
+ System.arraycopy(source, r.start(), tmp, 0, r.length());
+ }
+ splits[i] = tmp;
+ }
+ return splits;
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ */
+ public static List<Range> splitRanges(byte[] source, byte[] separator) {
+ return splitRanges(source, separator, -1);
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ *
+ * @param source the source data
+ * @param separator the separator pattern to look for
+ * @param limit the maximum number of splits to identify in the source
+ */
+ public static List<Range> splitRanges(byte[] source, byte[] separator,
+ int limit) {
+ List<Range> segments = new ArrayList<Range>();
+ if ((source == null) || (separator == null)) {
+ return segments;
+ }
+ int start = 0;
+ itersource: for (int i = 0; i < source.length; i++) {
+ for (int j = 0; j < separator.length; j++) {
+ if (source[i + j] != separator[j]) {
+ continue itersource;
+ }
+ }
+ // all separator elements matched
+ if (limit > 0 && segments.size() >= (limit - 1)) {
+ // everything else goes in one final segment
+ break;
+ }
+
+ segments.add(new Range(start, i));
+ start = i + separator.length;
+ // i will be incremented again in outer for loop
+ i += separator.length - 1;
+ }
+ // add in remaining to a final range
+ if (start <= source.length) {
+ segments.add(new Range(start, source.length));
+ }
+ return segments;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
new file mode 100644
index 0000000..64a11f8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.hbase.client.BufferedMutator;
+
+/**
+ * Just a typed wrapper around {@link BufferedMutator} used to ensure that
+ * columns can write only to the table mutator for the right table.
+ */
+public interface TypedBufferedMutator<T> extends BufferedMutator {
+ // This class is intentionally left (almost) blank
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
new file mode 100644
index 0000000..32577fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
new file mode 100644
index 0000000..90da966
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies fully qualified columns for the {@link EntityTable}.
+ */
+public enum EntityColumn implements Column<EntityTable> {
+
+ /**
+ * Identifier for the entity.
+ */
+ ID(EntityColumnFamily.INFO, "id"),
+
+ /**
+ * The type of entity
+ */
+ TYPE(EntityColumnFamily.INFO, "type"),
+
+ /**
+ * When the entity was created.
+ */
+ CREATED_TIME(EntityColumnFamily.INFO, "created_time"),
+
+ /**
+ * When it was modified.
+ */
+ MODIFIED_TIME(EntityColumnFamily.INFO, "modified_time"),
+
+ /**
+ * The version of the flow that this entity belongs to.
+ */
+ FLOW_VERSION(EntityColumnFamily.INFO, "flow_version");
+
+ private final ColumnHelper<EntityTable> column;
+ private final ColumnFamily<EntityTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+
+ private EntityColumn(ColumnFamily<EntityTable> columnFamily,
+ String columnQualifier) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.column = new ColumnHelper<EntityTable>(columnFamily);
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<EntityTable> tableMutator, Long timestamp,
+ Object inputValue) throws IOException {
+ column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
+ inputValue);
+ }
+
+ public Object readResult(Result result) throws IOException {
+ return column.readResult(result, columnQualifierBytes);
+ }
+
+ /**
+ * Retrieve an {@link EntityColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(x) == columnFor(y)} if
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnQualifier Name of the column to retrieve
+ * @return the corresponding {@link EntityColumn} or null
+ */
+ public static final EntityColumn columnFor(String columnQualifier) {
+
+ // Match column based on value, assume column family matches.
+ for (EntityColumn ec : EntityColumn.values()) {
+ // Find a match based only on name.
+ if (ec.getColumnQualifier().equals(columnQualifier)) {
+ return ec;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link EntityColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)}
+ * if and only if {@code a.equals(b) & x.equals(y)} or
+ * {@code (x == y == null)}
+ *
+ * @param columnFamily The columnFamily for which to retrieve the column.
+ * @param name Name of the column to retrieve
+ * @return the corresponding {@link EntityColumn} or null if both arguments
+ * don't match.
+ */
+ public static final EntityColumn columnFor(EntityColumnFamily columnFamily,
+ String name) {
+
+ for (EntityColumn ec : EntityColumn.values()) {
+ // Find a match based column family and on name.
+ if (ec.columnFamily.equals(columnFamily)
+ && ec.getColumnQualifier().equals(name)) {
+ return ec;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
new file mode 100644
index 0000000..8a95d12
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the entity table column families.
+ */
+public enum EntityColumnFamily implements ColumnFamily<EntityTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i"),
+
+ /**
+ * Configurations are in a separate column family for two reasons: a) the size
+ * of the config values can be very large and b) we expect that config values
+ * are often separately accessed from other metrics and info columns.
+ */
+ CONFIGS("c"),
+
+ /**
+ * Metrics have a separate column family, because they have a separate TTL.
+ */
+ METRICS("m");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ private EntityColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
new file mode 100644
index 0000000..4459868
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies partially qualified columns for the entity table.
+ */
+public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
+
+ /**
+ * To store TimelineEntity getIsRelatedToEntities values.
+ */
+ IS_RELATED_TO(EntityColumnFamily.INFO, "s"),
+
+ /**
+ * To store TimelineEntity getRelatesToEntities values.
+ */
+ RELATES_TO(EntityColumnFamily.INFO, "r"),
+
+ /**
+ * Lifecycle events for an entity
+ */
+ EVENT(EntityColumnFamily.INFO, "e"),
+
+ /**
+ * Config column stores configuration with config key as the column name.
+ */
+ CONFIG(EntityColumnFamily.CONFIGS, null),
+
+ /**
+ * Metrics are stored with the metric name as the column name.
+ */
+ METRIC(EntityColumnFamily.METRICS, null);
+
+ private final ColumnHelper<EntityTable> column;
+ private final ColumnFamily<EntityTable> columnFamily;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ */
+ private EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
+ String columnPrefix) {
+ column = new ColumnHelper<EntityTable>(columnFamily);
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
+ }
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ */
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<EntityTable> tableMutator, String qualifier,
+ Long timestamp, Object inputValue) throws IOException {
+
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
+ */
+ public Object readResult(Result result, String qualifier) throws IOException {
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+ return column.readResult(result, columnQualifier);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResults(org.apache.hadoop.hbase.client.Result)
+ */
+ public Map<String, Object> readResults(Result result) throws IOException {
+ return column.readResults(result, columnPrefixBytes);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readTimeseriesResults(org.apache.hadoop.hbase.client.Result)
+ */
+ public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
+ Result result) throws IOException {
+ return column.readTimeseriesResults(result, columnPrefixBytes);
+ }
+
+ /**
+ * Retrieve an {@link EntityColumnPrefix} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(x) == columnFor(y)} if
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnPrefix Name of the column to retrieve
+ * @return the corresponding {@link EntityColumnPrefix} or null
+ */
+ public static final EntityColumnPrefix columnFor(String columnPrefix) {
+
+ // Match column based on value, assume column family matches.
+ for (EntityColumnPrefix ecp : EntityColumnPrefix.values()) {
+ // Find a match based only on name.
+ if (ecp.getColumnPrefix().equals(columnPrefix)) {
+ return ecp;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link EntityColumnPrefix} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)}
+ * if and only if {@code (x == y == null)} or
+ * {@code a.equals(b) & x.equals(y)}
+ *
+ * @param columnFamily The columnFamily for which to retrieve the column.
+ * @param columnPrefix Name of the column to retrieve
+ * @return the corresponding {@link EntityColumnPrefix} or null if both
+ * arguments don't match.
+ */
+ public static final EntityColumnPrefix columnFor(
+ EntityColumnFamily columnFamily, String columnPrefix) {
+
+ // TODO: needs unit test to confirm and need to update javadoc to explain
+ // null prefix case.
+
+ for (EntityColumnPrefix ecp : EntityColumnPrefix.values()) {
+ // Find a match based column family and on name.
+ if (ecp.columnFamily.equals(columnFamily)
+ && (((columnPrefix == null) && (ecp.getColumnPrefix() == null)) || (ecp
+ .getColumnPrefix().equals(columnPrefix)))) {
+ return ecp;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
new file mode 100644
index 0000000..61958c2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the entity table.
+ */
+public class EntityRowKey {
+ // TODO: more methods are needed for this class.
+
+ // TODO: API needs to be cleaned up.
+
+ /**
+ * Constructs a row key prefix for the entity table as follows:
+ * {@code userName!clusterId!flowId!flowRunId!AppId}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @param appId
+ * @return byte array with the row key prefix
+ */
+ public static byte[] getRowKeyPrefix(String clusterId, String userId,
+ String flowId, Long flowRunId, String appId) {
+ byte[] first =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
+ flowId));
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ byte[] second = Bytes.toBytes(EntityRowKey.invert(flowRunId));
+ byte[] third = Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId));
+ return Separator.QUALIFIERS.join(first, second, third);
+ }
+
+ /**
+ * Constructs a row key prefix for the entity table as follows:
+ * {@code userName!clusterId!flowId!flowRunId!AppId}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @param appId
+ * @return byte array with the row key prefix
+ */
+ public static byte[] getRowKey(String clusterId, String userId,
+ String flowId, Long flowRunId, String appId, TimelineEntity te) {
+ byte[] first =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
+ flowId));
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ byte[] second = Bytes.toBytes(EntityRowKey.invert(flowRunId));
+ byte[] third =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, te.getType(),
+ te.getId()));
+ return Separator.QUALIFIERS.join(first, second, third);
+ }
+
+ /**
+ * Converts a timestamp into it's inverse timestamp to be used in (row) keys
+ * where we want to have the most recent timestamp in the top of the table
+ * (scans start at the most recent timestamp first).
+ *
+ * @param key value to be inverted so that the latest version will be first in
+ * a scan.
+ * @return inverted long
+ */
+ public static long invert(Long key) {
+ return Long.MAX_VALUE - key;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
new file mode 100644
index 0000000..61f7c4c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineEntitySchemaConstants;
+
+/**
+ * The entity table as column families info, config and metrics. Info stores
+ * information about a timeline entity object config stores configuration data
+ * of a timeline entity object metrics stores the metrics of a timeline entity
+ * object
+ *
+ * Example entity table record:
+ *
+ * <pre>
+ * |--------------------------------------------------------------------|
+ * | Row | Column Family | Column Family| Column Family|
+ * | key | info | metrics | config |
+ * |--------------------------------------------------------------------|
+ * | userName! | id:entityId | metricId1: | configKey1: |
+ * | clusterId! | | metricValue1 | configValue1 |
+ * | flowId! | type:entityType | @timestamp1 | |
+ * | flowRunId! | | | configKey2: |
+ * | AppId! | created_time: | metriciD1: | configValue2 |
+ * | entityType!| 1392993084018 | metricValue2 | |
+ * | entityId | | @timestamp2 | |
+ * | | modified_time: | | |
+ * | | 1392995081012 | metricId2: | |
+ * | | | metricValue1 | |
+ * | | r!relatesToKey: | @timestamp2 | |
+ * | | id3?id4?id5 | | |
+ * | | | | |
+ * | | s!isRelatedToKey | | |
+ * | | id7?id9?id6 | | |
+ * | | | | |
+ * | | e!eventId?eventInfoKey: | | |
+ * | | eventInfoValue | | |
+ * | | | | |
+ * | | flowVersion: | | |
+ * | | versionValue | | |
+ * |--------------------------------------------------------------------|
+ * </pre>
+ */
+public class EntityTable extends BaseTable<EntityTable> {
+ /** entity prefix */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".entity";
+
+ /** config param name that specifies the entity table name */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /**
+ * config param name that specifies the TTL for metrics column family in
+ * entity table
+ */
+ private static final String METRICS_TTL_CONF_NAME = PREFIX
+ + ".table.metrics.ttl";
+
+ /** default value for entity table name */
+ private static final String DEFAULT_TABLE_NAME = "timelineservice.entity";
+
+ /** default TTL is 30 days for metrics timeseries */
+ private static final int DEFAULT_METRICS_TTL = 2592000;
+
+ /** default max number of versions */
+ private static final int DEFAULT_METRICS_MAX_VERSIONS = 1000;
+
+ private static final Log LOG = LogFactory.getLog(EntityTable.class);
+
+ public EntityTable() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
+ * (org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor entityTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(EntityColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ entityTableDescp.addFamily(infoCF);
+
+ HColumnDescriptor configCF =
+ new HColumnDescriptor(EntityColumnFamily.CONFIGS.getBytes());
+ configCF.setBloomFilterType(BloomType.ROWCOL);
+ configCF.setBlockCacheEnabled(true);
+ entityTableDescp.addFamily(configCF);
+
+ HColumnDescriptor metricsCF =
+ new HColumnDescriptor(EntityColumnFamily.METRICS.getBytes());
+ entityTableDescp.addFamily(metricsCF);
+ metricsCF.setBlockCacheEnabled(true);
+ // always keep 1 version (the latest)
+ metricsCF.setMinVersions(1);
+ metricsCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
+ metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
+ DEFAULT_METRICS_TTL));
+ entityTableDescp
+ .setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineEntitySchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(entityTableDescp,
+ TimelineEntitySchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+
+ /**
+ * @param metricsTTL time to live parameter for the metricss in this table.
+ * @param hbaseConf configururation in which to set the metrics TTL config
+ * variable.
+ */
+ public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
+ hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
new file mode 100644
index 0000000..26f1cc5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index f999b4d..6abf240 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -18,43 +18,41 @@
package org.apache.hadoop.yarn.server.timelineservice.storage;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.Set;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertNotNull;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-import org.junit.BeforeClass;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
/**
- * Unit test HBaseTimelineWriterImpl
- * YARN 3411
- *
* @throws Exception
*/
public class TestHBaseTimelineWriterImpl {
@@ -69,12 +67,8 @@ public class TestHBaseTimelineWriterImpl {
}
private static void createSchema() throws IOException {
- byte[][] families = new byte[3][];
- families[0] = EntityColumnFamily.INFO.getInBytes();
- families[1] = EntityColumnFamily.CONFIG.getInBytes();
- families[2] = EntityColumnFamily.METRICS.getInBytes();
- TimelineSchemaCreator.createTimelineEntityTable(util.getHBaseAdmin(),
- util.getConfiguration());
+ new EntityTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
}
@Test
@@ -151,18 +145,15 @@ public class TestHBaseTimelineWriterImpl {
// scan the table and see that entity exists
Scan s = new Scan();
- byte[] startRow = TimelineWriterUtils.getRowKeyPrefix(cluster, user, flow,
- runid, appName);
+ byte[] startRow =
+ EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
s.setStartRow(startRow);
s.setMaxVersions(Integer.MAX_VALUE);
- ResultScanner scanner = null;
- TableName entityTableName = TableName
- .valueOf(TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME);
Connection conn = ConnectionFactory.createConnection(c1);
- Table entityTable = conn.getTable(entityTableName);
+ ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+
int rowCount = 0;
int colCount = 0;
- scanner = entityTable.getScanner(s);
for (Result result : scanner) {
if (result != null && !result.isEmpty()) {
rowCount++;
@@ -172,37 +163,77 @@ public class TestHBaseTimelineWriterImpl {
entity));
// check info column family
- NavigableMap<byte[], byte[]> infoValues = result
- .getFamilyMap(EntityColumnFamily.INFO.getInBytes());
- String id1 = TimelineWriterUtils.getValueAsString(
- EntityColumnDetails.ID.getInBytes(), infoValues);
+ String id1 = EntityColumn.ID.readResult(result).toString();
assertEquals(id, id1);
- String type1 = TimelineWriterUtils.getValueAsString(
- EntityColumnDetails.TYPE.getInBytes(), infoValues);
+
+ String type1 = EntityColumn.TYPE.readResult(result).toString();
assertEquals(type, type1);
- Long cTime1 = TimelineWriterUtils.getValueAsLong(
- EntityColumnDetails.CREATED_TIME.getInBytes(), infoValues);
+
+ Number val = (Number) EntityColumn.CREATED_TIME.readResult(result);
+ Long cTime1 = val.longValue();
assertEquals(cTime1, cTime);
- Long mTime1 = TimelineWriterUtils.getValueAsLong(
- EntityColumnDetails.MODIFIED_TIME.getInBytes(), infoValues);
+
+ val = (Number) EntityColumn.MODIFIED_TIME.readResult(result);
+ Long mTime1 = val.longValue();
assertEquals(mTime1, mTime);
- checkRelatedEntities(isRelatedTo, infoValues,
- EntityColumnDetails.PREFIX_IS_RELATED_TO.getInBytes());
- checkRelatedEntities(relatesTo, infoValues,
- EntityColumnDetails.PREFIX_RELATES_TO.getInBytes());
-
- // check config column family
- NavigableMap<byte[], byte[]> configValuesResult = result
- .getFamilyMap(EntityColumnFamily.CONFIG.getInBytes());
- checkConfigs(configValuesResult, conf);
-
- NavigableMap<byte[], byte[]> metricsResult = result
- .getFamilyMap(EntityColumnFamily.METRICS.getInBytes());
- checkMetricsSizeAndKey(metricsResult, metrics);
- List<Cell> metricCells = result.getColumnCells(
- EntityColumnFamily.METRICS.getInBytes(),
- Bytes.toBytes(m1.getId()));
- checkMetricsTimeseries(metricCells, m1);
+
+ // Remember isRelatedTo is of type Map<String, Set<String>>
+ for (String isRelatedToKey : isRelatedTo.keySet()) {
+ Object isRelatedToValue =
+ EntityColumnPrefix.IS_RELATED_TO.readResult(result,
+ isRelatedToKey);
+ String compoundValue = isRelatedToValue.toString();
+ // id7?id9?id6
+ Set<String> isRelatedToValues =
+ new HashSet<String>(
+ Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(isRelatedTo.get(isRelatedToKey).size(),
+ isRelatedToValues.size());
+ for (String v : isRelatedTo.get(isRelatedToKey)) {
+ assertTrue(isRelatedToValues.contains(v));
+ }
+ }
+
+ // RelatesTo
+ for (String relatesToKey : relatesTo.keySet()) {
+ String compoundValue =
+ EntityColumnPrefix.RELATES_TO.readResult(result, relatesToKey)
+ .toString();
+ // id3?id4?id5
+ Set<String> relatesToValues =
+ new HashSet<String>(
+ Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(relatesTo.get(relatesToKey).size(),
+ relatesToValues.size());
+ for (String v : relatesTo.get(relatesToKey)) {
+ assertTrue(relatesToValues.contains(v));
+ }
+ }
+
+ // Configuration
+ Map<String, Object> configColumns =
+ EntityColumnPrefix.CONFIG.readResults(result);
+ assertEquals(conf.size(), configColumns.size());
+ for (String configItem : conf.keySet()) {
+ assertEquals(conf.get(configItem), configColumns.get(configItem));
+ }
+
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ EntityColumnPrefix.METRIC.readTimeseriesResults(result);
+
+ NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
+ // We got metrics back
+ assertNotNull(metricMap);
+ // Same number of metrics as we wrote
+ assertEquals(metricValues.entrySet().size(), metricMap.entrySet()
+ .size());
+
+ // Iterate over original metrics and confirm that they are present
+ // here.
+ for (Entry<Long, Number> metricEntry : metricValues.entrySet()) {
+ assertEquals(metricEntry.getValue(),
+ metricMap.get(metricEntry.getKey()));
+ }
}
}
assertEquals(1, rowCount);
@@ -212,80 +243,77 @@ public class TestHBaseTimelineWriterImpl {
hbi.stop();
hbi.close();
}
- }
-
- private void checkMetricsTimeseries(List<Cell> metricCells,
- TimelineMetric m1) throws IOException {
- Map<Long, Number> timeseries = m1.getValues();
- assertEquals(timeseries.size(), metricCells.size());
- for (Cell c1 : metricCells) {
- assertTrue(timeseries.containsKey(c1.getTimestamp()));
- assertEquals(GenericObjectMapper.read(CellUtil.cloneValue(c1)),
- timeseries.get(c1.getTimestamp()));
- }
- }
-
- private void checkMetricsSizeAndKey(
- NavigableMap<byte[], byte[]> metricsResult, Set<TimelineMetric> metrics) {
- assertEquals(metrics.size(), metricsResult.size());
- for (TimelineMetric m1 : metrics) {
- byte[] key = Bytes.toBytes(m1.getId());
- assertTrue(metricsResult.containsKey(key));
- }
- }
-
- private void checkConfigs(NavigableMap<byte[], byte[]> configValuesResult,
- Map<String, String> conf) throws IOException {
-
- assertEquals(conf.size(), configValuesResult.size());
- byte[] columnName;
- for (String key : conf.keySet()) {
- columnName = Bytes.toBytes(key);
- assertTrue(configValuesResult.containsKey(columnName));
- byte[] value = configValuesResult.get(columnName);
- assertNotNull(value);
- assertEquals(conf.get(key), GenericObjectMapper.read(value));
- }
- }
- private void checkRelatedEntities(Map<String, Set<String>> isRelatedTo,
- NavigableMap<byte[], byte[]> infoValues, byte[] columnPrefix)
- throws IOException {
-
- for (String key : isRelatedTo.keySet()) {
- byte[] columnName = TimelineWriterUtils.join(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES, columnPrefix,
- Bytes.toBytes(key));
-
- byte[] value = infoValues.get(columnName);
- assertNotNull(value);
- String isRelatedToEntities = GenericObjectMapper.read(value).toString();
- assertNotNull(isRelatedToEntities);
- assertEquals(
- TimelineWriterUtils.getValueAsString(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR,
- isRelatedTo.get(key)), isRelatedToEntities);
- }
+ // Somewhat of a hack, not a separate test in order not to have to deal with
+ // test case order exectution.
+ testAdditionalEntity();
}
private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
String flow, Long runid, String appName, TimelineEntity te) {
- byte[][] rowKeyComponents = TimelineWriterUtils.split(rowKey,
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES);
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, -1);
assertTrue(rowKeyComponents.length == 7);
assertEquals(user, Bytes.toString(rowKeyComponents[0]));
assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.encodeRunId(runid),
- Bytes.toLong(rowKeyComponents[3]));
- assertEquals(TimelineWriterUtils.cleanse(appName), Bytes.toString(rowKeyComponents[4]));
+ assertEquals(EntityRowKey.invert(runid), Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
return true;
}
+ private void testAdditionalEntity() throws IOException {
+ TimelineEvent event = new TimelineEvent();
+ event.setId("foo_event_id");
+ event.setTimestamp(System.currentTimeMillis());
+ event.addInfo("foo_event", "test");
+
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId("attempt_1329348432655_0001_m_000008_18");
+ entity.setType("FOO_ATTEMPT");
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ String cluster = "cluster2";
+ String user = "user2";
+ String flow = "other_flow_name";
+ String flowVersion = "1111F01C2287BA";
+ long runid = 1009876543218L;
+ String appName = "some app name";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
+ hbi.stop();
+ // scan the table and see that entity exists
+ Scan s = new Scan();
+ byte[] startRow =
+ EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
+ s.setStartRow(startRow);
+ s.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+
+ int rowCount = 0;
+ for (Result result : scanner) {
+ if (result != null && !result.isEmpty()) {
+ rowCount++;
+ }
+ }
+ assertEquals(1, rowCount);
+
+ } finally {
+ hbi.stop();
+ hbi.close();
+ }
+ }
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
util.shutdownMiniCluster();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
new file mode 100644
index 0000000..8b25a83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.google.common.collect.Iterables;
+
+public class TestSeparator {
+
+ private static String villain = "Dr. Heinz Doofenshmirtz";
+ private static String special =
+ ". * | ? + ( ) [ ] { } ^ $ \\ \"";
+
+ /**
+ *
+ */
+ @Test
+ public void testEncodeDecodeString() {
+
+ for (Separator separator : Separator.values()) {
+ testEncodeDecode(separator, "");
+ testEncodeDecode(separator, " ");
+ testEncodeDecode(separator, "!");
+ testEncodeDecode(separator, "?");
+ testEncodeDecode(separator, "&");
+ testEncodeDecode(separator, "+");
+ testEncodeDecode(separator, "Dr.");
+ testEncodeDecode(separator, "Heinz");
+ testEncodeDecode(separator, "Doofenshmirtz");
+ testEncodeDecode(separator, villain);
+ testEncodeDecode(separator, special);
+
+ assertNull(separator.encode(null));
+
+ }
+ }
+
+ private void testEncodeDecode(Separator separator, String token) {
+ String encoded = separator.encode(token);
+ String decoded = separator.decode(encoded);
+ String msg = "token:" + token + " separator:" + separator + ".";
+ assertEquals(msg, token, decoded);
+ }
+
+ @Test
+ public void testEncodeDecode() {
+ testEncodeDecode("Dr.", Separator.QUALIFIERS);
+ testEncodeDecode("Heinz", Separator.QUALIFIERS, Separator.QUALIFIERS);
+ testEncodeDecode("Doofenshmirtz", Separator.QUALIFIERS, null,
+ Separator.QUALIFIERS);
+ testEncodeDecode("&Perry", Separator.QUALIFIERS, Separator.VALUES, null);
+ testEncodeDecode("the ", Separator.QUALIFIERS, Separator.SPACE);
+ testEncodeDecode("Platypus...", (Separator) null);
+ testEncodeDecode("The what now ?!?", Separator.QUALIFIERS,
+ Separator.VALUES, Separator.SPACE);
+
+ }
+
+ /**
+ * Simple test to encode and decode using the same separators and confirm that
+ * we end up with the same as what we started with.
+ *
+ * @param token
+ * @param separators
+ */
+ private static void testEncodeDecode(String token, Separator... separators) {
+ byte[] encoded = Separator.encode(token, separators);
+ String decoded = Separator.decode(encoded, separators);
+ assertEquals(token, decoded);
+ }
+
+ @Test
+ public void testJoinStripped() {
+ List<String> stringList = new ArrayList<String>(0);
+ stringList.add("nothing");
+
+ String joined = Separator.VALUES.joinEncoded(stringList);
+ Iterable<String> split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(stringList, split));
+
+ stringList = new ArrayList<String>(3);
+ stringList.add("a");
+ stringList.add("b?");
+ stringList.add("c");
+
+ joined = Separator.VALUES.joinEncoded(stringList);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(stringList, split));
+
+ String[] stringArray1 = { "else" };
+ joined = Separator.VALUES.joinEncoded(stringArray1);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray1), split));
+
+ String[] stringArray2 = { "d", "e?", "f" };
+ joined = Separator.VALUES.joinEncoded(stringArray2);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray2), split));
+
+ List<String> empty = new ArrayList<String>(0);
+ split = Separator.VALUES.splitEncoded(null);
+ assertTrue(Iterables.elementsEqual(empty, split));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3599c8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java
new file mode 100644
index 0000000..4f96f87
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.junit.Test;
+
+public class TestTimelineWriterUtils {
+
+ @Test
+ public void test() {
+ // TODO: implement a test for the remaining method in TimelineWriterUtils.
+ }
+
+}
[27/50] [abbrv] hadoop git commit: YARN-3906. Split the application
table from the entity table. Contributed by Sangjin Lee.
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/491b3023/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index ab02779..95f88d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -47,6 +48,10 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
@@ -60,7 +65,15 @@ import org.junit.BeforeClass;
import org.junit.Test;
/**
- * @throws Exception
+ * Various tests to test writing entities to HBase and reading them back from
+ * it.
+ *
+ * It uses a single HBase mini-cluster for all tests which is a little more
+ * realistic, and helps test correctness in the presence of other data.
+ *
+ * Each test uses a different cluster name to be able to handle its own data
+ * even if other records exist in the table. Use a different cluster name if
+ * you add a new test.
*/
public class TestHBaseTimelineWriterImpl {
@@ -78,6 +91,199 @@ public class TestHBaseTimelineWriterImpl {
.createTable(util.getHBaseAdmin(), util.getConfiguration());
new AppToFlowTable()
.createTable(util.getHBaseAdmin(), util.getConfiguration());
+ new ApplicationTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
+ }
+
+ @Test
+ public void testWriteApplicationToHBase() throws Exception {
+ TimelineEntities te = new TimelineEntities();
+ ApplicationEntity entity = new ApplicationEntity();
+ String id = "hello";
+ entity.setId(id);
+ Long cTime = 1425016501000L;
+ Long mTime = 1425026901000L;
+ entity.setCreatedTime(cTime);
+ entity.setModifiedTime(mTime);
+
+ // add the info map in Timeline Entity
+ Map<String, Object> infoMap = new HashMap<String, Object>();
+ infoMap.put("infoMapKey1", "infoMapValue1");
+ infoMap.put("infoMapKey2", 10);
+ entity.addInfo(infoMap);
+
+ // add the isRelatedToEntity info
+ String key = "task";
+ String value = "is_related_to_entity_id_here";
+ Set<String> isRelatedToSet = new HashSet<String>();
+ isRelatedToSet.add(value);
+ Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
+ isRelatedTo.put(key, isRelatedToSet);
+ entity.setIsRelatedToEntities(isRelatedTo);
+
+ // add the relatesTo info
+ key = "container";
+ value = "relates_to_entity_id_here";
+ Set<String> relatesToSet = new HashSet<String>();
+ relatesToSet.add(value);
+ value = "relates_to_entity_id_here_Second";
+ relatesToSet.add(value);
+ Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
+ relatesTo.put(key, relatesToSet);
+ entity.setRelatesToEntities(relatesTo);
+
+ // add some config entries
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put("config_param1", "value1");
+ conf.put("config_param2", "value2");
+ entity.addConfigs(conf);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 120000, 100000000);
+ metricValues.put(ts - 100000, 200000000);
+ metricValues.put(ts - 80000, 300000000);
+ metricValues.put(ts - 60000, 400000000);
+ metricValues.put(ts - 40000, 50000000000L);
+ metricValues.put(ts - 20000, 60000000000L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ te.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ String cluster = "cluster_test_write_app";
+ String user = "user1";
+ String flow = "some_flow_name";
+ String flowVersion = "AB7822C10F1111";
+ long runid = 1002345678919L;
+ hbi.write(cluster, user, flow, flowVersion, runid, id, te);
+ hbi.stop();
+
+ // retrieve the row
+ byte[] rowKey =
+ ApplicationRowKey.getRowKey(cluster, user, flow, runid, id);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Result result = new ApplicationTable().getResult(c1, conn, get);
+
+ assertTrue(result != null);
+ assertEquals(16, result.size());
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
+ id));
+
+ // check info column family
+ String id1 = ApplicationColumn.ID.readResult(result).toString();
+ assertEquals(id, id1);
+
+ Number val =
+ (Number) ApplicationColumn.CREATED_TIME.readResult(result);
+ Long cTime1 = val.longValue();
+ assertEquals(cTime1, cTime);
+
+ val = (Number) ApplicationColumn.MODIFIED_TIME.readResult(result);
+ Long mTime1 = val.longValue();
+ assertEquals(mTime1, mTime);
+
+ Map<String, Object> infoColumns =
+ ApplicationColumnPrefix.INFO.readResults(result);
+ assertEquals(infoMap.size(), infoColumns.size());
+ for (String infoItem : infoMap.keySet()) {
+ assertEquals(infoMap.get(infoItem), infoColumns.get(infoItem));
+ }
+
+ // Remember isRelatedTo is of type Map<String, Set<String>>
+ for (String isRelatedToKey : isRelatedTo.keySet()) {
+ Object isRelatedToValue =
+ ApplicationColumnPrefix.IS_RELATED_TO.readResult(result,
+ isRelatedToKey);
+ String compoundValue = isRelatedToValue.toString();
+ // id7?id9?id6
+ Set<String> isRelatedToValues =
+ new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(isRelatedTo.get(isRelatedToKey).size(),
+ isRelatedToValues.size());
+ for (String v : isRelatedTo.get(isRelatedToKey)) {
+ assertTrue(isRelatedToValues.contains(v));
+ }
+ }
+
+ // RelatesTo
+ for (String relatesToKey : relatesTo.keySet()) {
+ String compoundValue =
+ ApplicationColumnPrefix.RELATES_TO.readResult(result,
+ relatesToKey).toString();
+ // id3?id4?id5
+ Set<String> relatesToValues =
+ new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(relatesTo.get(relatesToKey).size(),
+ relatesToValues.size());
+ for (String v : relatesTo.get(relatesToKey)) {
+ assertTrue(relatesToValues.contains(v));
+ }
+ }
+
+ // Configuration
+ Map<String, Object> configColumns =
+ ApplicationColumnPrefix.CONFIG.readResults(result);
+ assertEquals(conf.size(), configColumns.size());
+ for (String configItem : conf.keySet()) {
+ assertEquals(conf.get(configItem), configColumns.get(configItem));
+ }
+
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
+
+ NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
+ // We got metrics back
+ assertNotNull(metricMap);
+ // Same number of metrics as we wrote
+ assertEquals(metricValues.entrySet().size(), metricMap.entrySet().size());
+
+ // Iterate over original metrics and confirm that they are present
+ // here.
+ for (Entry<Long, Number> metricEntry : metricValues.entrySet()) {
+ assertEquals(metricEntry.getValue(),
+ metricMap.get(metricEntry.getKey()));
+ }
+
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, id,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ id, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
+ } finally {
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
+ }
}
@Test
@@ -154,7 +360,7 @@ public class TestHBaseTimelineWriterImpl {
hbr = new HBaseTimelineReaderImpl();
hbr.init(c1);
hbr.start();
- String cluster = "cluster1";
+ String cluster = "cluster_test_write_entity";
String user = "user1";
String flow = "some_flow_name";
String flowVersion = "AB7822C10F1111";
@@ -268,7 +474,8 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(17, colCount);
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
appName, entity.getType(), null, null, null, null, null, null, null,
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
@@ -284,10 +491,6 @@ public class TestHBaseTimelineWriterImpl {
hbr.close();
}
}
-
- // Somewhat of a hack, not a separate test in order not to have to deal with
- // test case order exectution.
- testAdditionalEntity();
}
private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
@@ -299,14 +502,31 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(user, Bytes.toString(rowKeyComponents[0]));
assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.invert(runid), Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(TimelineWriterUtils.invert(runid),
+ Bytes.toLong(rowKeyComponents[3]));
assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
return true;
}
- private void testAdditionalEntity() throws IOException {
+ private boolean isApplicationRowKeyCorrect(byte[] rowKey, String cluster,
+ String user, String flow, Long runid, String appName) {
+
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, -1);
+
+ assertTrue(rowKeyComponents.length == 5);
+ assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
+ assertEquals(user, Bytes.toString(rowKeyComponents[1]));
+ assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
+ assertEquals(TimelineWriterUtils.invert(runid),
+ Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
+ return true;
+ }
+
+ @Test
+ public void testEvents() throws IOException {
TimelineEvent event = new TimelineEvent();
String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
event.setId(eventId);
@@ -333,7 +553,7 @@ public class TestHBaseTimelineWriterImpl {
hbr = new HBaseTimelineReaderImpl();
hbr.init(c1);
hbr.start();
- String cluster = "cluster2";
+ String cluster = "cluster_test_events";
String user = "user2";
String flow = "other_flow_name";
String flowVersion = "1111F01C2287BA";
@@ -341,50 +561,46 @@ public class TestHBaseTimelineWriterImpl {
String appName = "some app name";
hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
hbi.stop();
- // scan the table and see that entity exists
- Scan s = new Scan();
- byte[] startRow =
- EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
- s.setStartRow(startRow);
- Connection conn = ConnectionFactory.createConnection(c1);
- ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
-
- int rowCount = 0;
- for (Result result : scanner) {
- if (result != null && !result.isEmpty()) {
- rowCount++;
-
- // check the row key
- byte[] row1 = result.getRow();
- assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
- entity));
- Map<String, Object> eventsResult =
- EntityColumnPrefix.EVENT.readResults(result);
- // there should be only one event
- assertEquals(1, eventsResult.size());
- // key name for the event
- byte[] compoundColumnQualifierBytes =
- Separator.VALUES.join(Bytes.toBytes(eventId),
- Bytes.toBytes(TimelineWriterUtils.invert(expTs)),
- Bytes.toBytes(expKey));
- String valueKey = Bytes.toString(compoundColumnQualifierBytes);
- for (Map.Entry<String, Object> e :
- eventsResult.entrySet()) {
- // the value key must match
- assertEquals(valueKey, e.getKey());
- Object value = e.getValue();
- // there should be only one timestamp and value
- assertEquals(expVal, value.toString());
- }
- }
+ // retrieve the row
+ byte[] rowKey =
+ ApplicationRowKey.getRowKey(cluster, user, flow, runid, appName);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Result result = new ApplicationTable().getResult(c1, conn, get);
+
+ assertTrue(result != null);
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
+ appName));
+
+ Map<String, Object> eventsResult =
+ ApplicationColumnPrefix.EVENT.readResults(result);
+ // there should be only one event
+ assertEquals(1, eventsResult.size());
+ // key name for the event
+ byte[] compoundColumnQualifierBytes =
+ Separator.VALUES.join(Bytes.toBytes(eventId),
+ Bytes.toBytes(TimelineWriterUtils.invert(expTs)),
+ Bytes.toBytes(expKey));
+ String valueKey = Bytes.toString(compoundColumnQualifierBytes);
+ for (Map.Entry<String, Object> e : eventsResult.entrySet()) {
+ // the value key must match
+ assertEquals(valueKey, e.getKey());
+ Object value = e.getValue();
+ // there should be only one timestamp and value
+ assertEquals(expVal, value.toString());
}
- assertEquals(1, rowCount);
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
TimelineEntity e2 = hbr.getEntity(user, cluster, null, null, appName,
- entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
appName, entity.getType(), null, null, null, null, null, null, null,
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
@@ -410,7 +626,7 @@ public class TestHBaseTimelineWriterImpl {
}
@Test
- public void testAdditionalEntityEmptyEventInfo() throws IOException {
+ public void testEventsWithEmptyInfo() throws IOException {
TimelineEvent event = new TimelineEvent();
String eventId = "foo_event_id";
event.setId(eventId);
@@ -435,7 +651,7 @@ public class TestHBaseTimelineWriterImpl {
hbr = new HBaseTimelineReaderImpl();
hbr.init(c1);
hbr.start();
- String cluster = "cluster_emptyeventkey";
+ String cluster = "cluster_test_empty_eventkey";
String user = "user_emptyeventkey";
String flow = "other_flow_name";
String flowVersion = "1111F01C2287BA";
@@ -487,7 +703,8 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(1, rowCount);
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
appName, entity.getType(), null, null, null, null, null, null, null,
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
[29/50] [abbrv] hadoop git commit: YARN-3049. [Storage
Implementation] Implement storage reader interface to fetch raw data from
HBase backend (Zhijie Shen via sjlee)
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecbdad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index fd5643d..ab02779 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -38,11 +39,15 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
@@ -71,6 +76,8 @@ public class TestHBaseTimelineWriterImpl {
private static void createSchema() throws IOException {
new EntityTable()
.createTable(util.getHBaseAdmin(), util.getConfiguration());
+ new AppToFlowTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
}
@Test
@@ -138,10 +145,15 @@ public class TestHBaseTimelineWriterImpl {
te.addEntity(entity);
HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
try {
Configuration c1 = util.getConfiguration();
hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
String cluster = "cluster1";
String user = "user1";
String flow = "some_flow_name";
@@ -255,9 +267,22 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(1, rowCount);
assertEquals(17, colCount);
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
} finally {
- hbi.stop();
- hbi.close();
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
}
// Somewhat of a hack, not a separate test in order not to have to deal with
@@ -283,7 +308,7 @@ public class TestHBaseTimelineWriterImpl {
private void testAdditionalEntity() throws IOException {
TimelineEvent event = new TimelineEvent();
- String eventId = "foo_event_id";
+ String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
event.setId(eventId);
Long expTs = 1436512802000L;
event.setTimestamp(expTs);
@@ -291,19 +316,23 @@ public class TestHBaseTimelineWriterImpl {
Object expVal = "test";
event.addInfo(expKey, expVal);
- final TimelineEntity entity = new TimelineEntity();
- entity.setId("attempt_1329348432655_0001_m_000008_18");
- entity.setType("FOO_ATTEMPT");
+ final TimelineEntity entity = new ApplicationEntity();
+ entity.setId(ApplicationId.newInstance(0, 1).toString());
entity.addEvent(event);
TimelineEntities entities = new TimelineEntities();
entities.addEntity(entity);
HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
try {
Configuration c1 = util.getConfiguration();
hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
String cluster = "cluster2";
String user = "user2";
String flow = "other_flow_name";
@@ -352,9 +381,31 @@ public class TestHBaseTimelineWriterImpl {
}
assertEquals(1, rowCount);
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ TimelineEntity e2 = hbr.getEntity(user, cluster, null, null, appName,
+ entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es2 = hbr.getEntities(user, cluster, null, null,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertNotNull(e2);
+ assertEquals(e1, e2);
+ assertEquals(1, es1.size());
+ assertEquals(1, es2.size());
+ assertEquals(es1, es2);
} finally {
- hbi.stop();
- hbi.close();
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
}
}
@@ -375,10 +426,15 @@ public class TestHBaseTimelineWriterImpl {
entities.addEntity(entity);
HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
try {
Configuration c1 = util.getConfiguration();
hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
String cluster = "cluster_emptyeventkey";
String user = "user_emptyeventkey";
String flow = "other_flow_name";
@@ -430,13 +486,21 @@ public class TestHBaseTimelineWriterImpl {
}
assertEquals(1, rowCount);
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
} finally {
hbi.stop();
hbi.close();
+ hbr.stop();;
+ hbr.close();
}
}
-
@AfterClass
public static void tearDownAfterClass() throws Exception {
util.shutdownMiniCluster();
[33/50] [abbrv] hadoop git commit: YARN-3045. Implement NM writing
container lifecycle events to Timeline Service v2. Contributed by
Naganarasimha G R.
Posted by sj...@apache.org.
YARN-3045. Implement NM writing container lifecycle events to Timeline Service v2. Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3fb80b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3fb80b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3fb80b9
Branch: refs/heads/YARN-2928-rebase
Commit: d3fb80b950fca3ed387de1f06618e4124bb7c0f4
Parents: 0fd93f1
Author: Junping Du <ju...@apache.org>
Authored: Tue Aug 18 04:31:45 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 16:13:21 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 16 +-
.../distributedshell/TestDistributedShell.java | 28 +-
.../hadoop/yarn/server/nodemanager/Context.java | 5 +
.../yarn/server/nodemanager/NodeManager.java | 13 +
.../containermanager/ContainerManagerImpl.java | 46 ++-
.../ApplicationContainerFinishedEvent.java | 17 +-
.../containermanager/container/Container.java | 3 +
.../container/ContainerImpl.java | 30 +-
.../monitor/ContainersMonitorImpl.java | 108 +-----
.../timelineservice/NMTimelineEvent.java | 31 ++
.../timelineservice/NMTimelineEventType.java | 24 ++
.../timelineservice/NMTimelinePublisher.java | 376 +++++++++++++++++++
.../nodemanager/TestNodeStatusUpdater.java | 24 +-
.../amrmproxy/BaseAMRMProxyTest.java | 9 +
.../containermanager/TestAuxServices.java | 4 +-
.../TestContainerManagerRecovery.java | 8 +
.../application/TestApplication.java | 7 +-
.../container/TestContainer.java | 2 +-
.../nodemanager/webapp/MockContainer.java | 6 +
.../nodemanager/webapp/TestNMWebServer.java | 3 +-
.../PerNodeTimelineCollectorsAuxService.java | 16 +-
...TestPerNodeTimelineCollectorsAuxService.java | 9 +
23 files changed, 646 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 374b254..e5c609a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -91,6 +91,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3904. Refactor timelineservice.storage to add support to online and
offline aggregation writers (Li Lu via sjlee)
+ YARN-3045. Implement NM writing container lifecycle events to Timeline
+ Service v2. (Naganarasimha G R via junping_du)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 691170e..62e60a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -110,6 +110,16 @@
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
+ <!-- Object cast is based on the event type -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.metrics.AbstractTimelineServicePublisher" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher$ApplicationEventHandler" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
<!-- Ignore intentional switch fallthroughs -->
<Match>
@@ -505,10 +515,4 @@
</Or>
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
-
- <!-- Object cast is based on the event type -->
- <Match>
- <Class name="org.apache.hadoop.yarn.server.resourcemanager.metrics.AbstractTimelineServicePublisher" />
- <Bug pattern="BC_UNCONFIRMED_CAST" />
- </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index b8a7abf..c89bee9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
@@ -203,7 +204,7 @@ public class TestDistributedShell {
testDSShell(false, "v2", false);
}
- public void testDSShell(boolean haveDomain, String timelineVersion,
+ private void testDSShell(boolean haveDomain, String timelineVersion,
boolean defaultFlow)
throws Exception {
String[] args = {
@@ -404,9 +405,32 @@ public class TestDistributedShell {
"container_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ "_01_000001"
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
- verifyEntityTypeFileExists(basePath,
+ File containerEntityFile = verifyEntityTypeFileExists(basePath,
TimelineEntityType.YARN_CONTAINER.toString(),
containerMetricsTimestampFileName);
+ Assert.assertEquals(
+ "Container created event needs to be published atleast once",
+ 1,
+ getNumOfStringOccurences(containerEntityFile,
+ ContainerMetricsConstants.CREATED_EVENT_TYPE));
+
+ // to avoid race condition of testcase, atleast check 4 times with sleep
+ // of 500ms
+ long numOfContainerFinishedOccurences = 0;
+ for (int i = 0; i < 4; i++) {
+ numOfContainerFinishedOccurences =
+ getNumOfStringOccurences(containerEntityFile,
+ ContainerMetricsConstants.FINISHED_EVENT_TYPE);
+ if (numOfContainerFinishedOccurences > 0) {
+ break;
+ } else {
+ Thread.sleep(500l);
+ }
+ }
+ Assert.assertEquals(
+ "Container finished event needs to be published atleast once",
+ 1,
+ numOfContainerFinishedOccurences);
// Verify RM posting Application life cycle Events are getting published
String appMetricsTimestampFileName =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 924860b..0b378a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
/**
@@ -96,4 +97,8 @@ public interface Context {
ConcurrentLinkedQueue<LogAggregationReport>
getLogAggregationStatusForApps();
+
+ void setNMTimelinePublisher(NMTimelinePublisher nmMetricsPublisher);
+
+ NMTimelinePublisher getNMTimelinePublisher();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index f64d7fe..8a9d1fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreServic
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -473,6 +474,8 @@ public class NodeManager extends CompositeService
private final ConcurrentLinkedQueue<LogAggregationReport>
logAggregationReportForApps;
+ private NMTimelinePublisher nmTimelinePublisher;
+
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager,
LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
@@ -615,6 +618,16 @@ public class NodeManager extends CompositeService
Map<ApplicationId, String> newRegisteredCollectors) {
this.registeredCollectors.putAll(newRegisteredCollectors);
}
+
+ @Override
+ public void setNMTimelinePublisher(NMTimelinePublisher nmMetricsPublisher) {
+ this.nmTimelinePublisher = nmMetricsPublisher;
+ }
+
+ @Override
+ public NMTimelinePublisher getNMTimelinePublisher() {
+ return nmTimelinePublisher;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index da79446..12aab79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -125,6 +125,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.SignalContainersLauncherEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadService;
@@ -142,6 +143,7 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.Re
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -188,6 +190,8 @@ public class ContainerManagerImpl extends CompositeService implements
private long waitForContainersOnShutdownMillis;
+ private final NMTimelinePublisher nmMetricsPublisher;
+
public ContainerManagerImpl(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
NodeManagerMetrics metrics, LocalDirsHandlerService dirsHandler) {
@@ -214,6 +218,8 @@ public class ContainerManagerImpl extends CompositeService implements
auxiliaryServices.registerServiceListener(this);
addService(auxiliaryServices);
+ nmMetricsPublisher = createNMTimelinePublisher(context);
+ context.setNMTimelinePublisher(nmMetricsPublisher);
this.containersMonitor =
new ContainersMonitorImpl(exec, dispatcher, this.context);
addService(this.containersMonitor);
@@ -222,13 +228,16 @@ public class ContainerManagerImpl extends CompositeService implements
new ContainerEventDispatcher());
dispatcher.register(ApplicationEventType.class,
new ApplicationEventDispatcher());
- dispatcher.register(LocalizationEventType.class, rsrcLocalizationSrvc);
+ dispatcher.register(LocalizationEventType.class,
+ new LocalizationEventHandlerWrapper(rsrcLocalizationSrvc,
+ nmMetricsPublisher));
dispatcher.register(AuxServicesEventType.class, auxiliaryServices);
dispatcher.register(ContainersMonitorEventType.class, containersMonitor);
dispatcher.register(ContainersLauncherEventType.class, containersLauncher);
addService(dispatcher);
+
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
@@ -351,7 +360,7 @@ public class ContainerManagerImpl extends CompositeService implements
Container container = new ContainerImpl(getConfig(), dispatcher,
context.getNMStateStore(), req.getContainerLaunchContext(),
credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(),
- rcs.getDiagnostics(), rcs.getKilled(), rcs.getCapability());
+ rcs.getDiagnostics(), rcs.getKilled(), rcs.getCapability(), context);
context.getContainers().put(containerId, container);
dispatcher.getEventHandler().handle(
new ApplicationContainerInitEvent(container));
@@ -413,6 +422,13 @@ public class ContainerManagerImpl extends CompositeService implements
return new SharedCacheUploadService();
}
+ @VisibleForTesting
+ protected NMTimelinePublisher createNMTimelinePublisher(Context context) {
+ NMTimelinePublisher nmTimelinePublisherLocal = new NMTimelinePublisher(context);
+ addIfService(nmTimelinePublisherLocal);
+ return nmTimelinePublisherLocal;
+ }
+
protected ContainersLauncher createContainersLauncher(Context context,
ContainerExecutor exec) {
return new ContainersLauncher(context, this.dispatcher, exec, dirsHandler, this);
@@ -910,7 +926,7 @@ public class ContainerManagerImpl extends CompositeService implements
Container container =
new ContainerImpl(getConfig(), this.dispatcher,
context.getNMStateStore(), launchContext,
- credentials, metrics, containerTokenIdentifier);
+ credentials, metrics, containerTokenIdentifier, context);
ApplicationId applicationID =
containerId.getApplicationAttemptId().getApplicationId();
if (context.getContainers().putIfAbsent(containerId, container) != null) {
@@ -952,9 +968,9 @@ public class ContainerManagerImpl extends CompositeService implements
logAggregationContext));
}
- this.context.getNMStateStore().storeContainer(containerId, request);
dispatcher.getEventHandler().handle(
new ApplicationContainerInitEvent(container));
+ this.context.getNMStateStore().storeContainer(containerId, request);
this.context.getContainerTokenSecretManager().startContainerSuccessful(
containerTokenIdentifier);
@@ -1291,6 +1307,7 @@ public class ContainerManagerImpl extends CompositeService implements
Container c = containers.get(event.getContainerID());
if (c != null) {
c.handle(event);
+ nmMetricsPublisher.publishContainerEvent(event);
} else {
LOG.warn("Event " + event + " sent to absent container " +
event.getContainerID());
@@ -1299,7 +1316,6 @@ public class ContainerManagerImpl extends CompositeService implements
}
class ApplicationEventDispatcher implements EventHandler<ApplicationEvent> {
-
@Override
public void handle(ApplicationEvent event) {
Application app =
@@ -1307,6 +1323,7 @@ public class ContainerManagerImpl extends CompositeService implements
event.getApplicationID());
if (app != null) {
app.handle(event);
+ nmMetricsPublisher.publishApplicationEvent(event);
} else {
LOG.warn("Event " + event + " sent to absent application "
+ event.getApplicationID());
@@ -1314,6 +1331,25 @@ public class ContainerManagerImpl extends CompositeService implements
}
}
+ private static final class LocalizationEventHandlerWrapper implements
+ EventHandler<LocalizationEvent> {
+
+ private EventHandler<LocalizationEvent> origLocalizationEventHandler;
+ private NMTimelinePublisher timelinePublisher;
+
+ LocalizationEventHandlerWrapper(EventHandler<LocalizationEvent> handler,
+ NMTimelinePublisher publisher) {
+ this.origLocalizationEventHandler = handler;
+ this.timelinePublisher = publisher;
+ }
+
+ @Override
+ public void handle(LocalizationEvent event) {
+ origLocalizationEventHandler.handle(event);
+ timelinePublisher.publishLocalizationEvent(event);
+ }
+ }
+
@SuppressWarnings("unchecked")
@Override
public void handle(ContainerManagerEvent event) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java
index 6b8007f..9cd34cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java
@@ -19,18 +19,23 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.application;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
public class ApplicationContainerFinishedEvent extends ApplicationEvent {
- private ContainerId containerID;
+ private ContainerStatus containerStatus;
- public ApplicationContainerFinishedEvent(
- ContainerId containerID) {
- super(containerID.getApplicationAttemptId().getApplicationId(),
+ public ApplicationContainerFinishedEvent(ContainerStatus containerStatus) {
+ super(containerStatus.getContainerId().getApplicationAttemptId().getApplicationId(),
ApplicationEventType.APPLICATION_CONTAINER_FINISHED);
- this.containerID = containerID;
+ this.containerStatus = containerStatus;
}
public ContainerId getContainerID() {
- return this.containerID;
+ return containerStatus.getContainerId();
}
+
+ public ContainerStatus getContainerStatus() {
+ return containerStatus;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 1d2ec56..aac88fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
@@ -57,4 +58,6 @@ public interface Container extends EventHandler<ContainerEvent> {
String toString();
+ Priority getPriority();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index eff2188..fb5f451 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -44,12 +44,14 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@@ -70,6 +72,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
@@ -125,11 +128,12 @@ public class ContainerImpl implements Container {
RecoveredContainerStatus.REQUESTED;
// whether container was marked as killed after recovery
private boolean recoveredAsKilled = false;
+ private Context context;
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
NMStateStoreService stateStore, ContainerLaunchContext launchContext,
Credentials creds, NodeManagerMetrics metrics,
- ContainerTokenIdentifier containerTokenIdentifier) {
+ ContainerTokenIdentifier containerTokenIdentifier, Context context) {
this.daemonConf = conf;
this.dispatcher = dispatcher;
this.stateStore = stateStore;
@@ -144,8 +148,8 @@ public class ContainerImpl implements Container {
ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
this.readLock = readWriteLock.readLock();
this.writeLock = readWriteLock.writeLock();
-
stateMachine = stateMachineFactory.make(this);
+ this.context = context;
}
// constructor for a recovered container
@@ -154,9 +158,10 @@ public class ContainerImpl implements Container {
Credentials creds, NodeManagerMetrics metrics,
ContainerTokenIdentifier containerTokenIdentifier,
RecoveredContainerStatus recoveredStatus, int exitCode,
- String diagnostics, boolean wasKilled, Resource recoveredCapability) {
+ String diagnostics, boolean wasKilled, Resource recoveredCapability,
+ Context context) {
this(conf, dispatcher, stateStore, launchContext, creds, metrics,
- containerTokenIdentifier);
+ containerTokenIdentifier, context);
this.recoveredStatus = recoveredStatus;
this.exitCode = exitCode;
this.recoveredAsKilled = wasKilled;
@@ -372,6 +377,10 @@ public class ContainerImpl implements Container {
}
}
+ public NMTimelinePublisher getNMTimelinePublisher() {
+ return context.getNMTimelinePublisher();
+ }
+
@Override
public String getUser() {
this.readLock.lock();
@@ -483,7 +492,10 @@ public class ContainerImpl implements Container {
// Inform the application
@SuppressWarnings("rawtypes")
EventHandler eventHandler = dispatcher.getEventHandler();
- eventHandler.handle(new ApplicationContainerFinishedEvent(containerId));
+
+ ContainerStatus containerStatus = cloneAndGetContainerStatus();
+ eventHandler.handle(new ApplicationContainerFinishedEvent(containerStatus));
+
// Remove the container from the resource-monitor
eventHandler.handle(new ContainerStopMonitoringEvent(containerId));
// Tell the logService too
@@ -985,7 +997,8 @@ public class ContainerImpl implements Container {
public void transition(ContainerImpl container, ContainerEvent event) {
container.metrics.releaseContainer(container.resource);
container.sendFinishedEvents();
- //if the current state is NEW it means the CONTAINER_INIT was never
+
+ // if the current state is NEW it means the CONTAINER_INIT was never
// sent for the event, thus no need to send the CONTAINER_STOP
if (container.getCurrentState()
!= org.apache.hadoop.yarn.api.records.ContainerState.NEW) {
@@ -1171,4 +1184,9 @@ public class ContainerImpl implements Container {
LocalResourceRequest resource) {
return container.resourcesUploadPolicies.get(resource);
}
+
+ @Override
+ public Priority getPriority() {
+ return containerTokenIdentifier.getPriority();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 6e00e18..dfa32ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -18,13 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
-import java.io.IOException;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
@@ -36,25 +32,20 @@ import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class ContainersMonitorImpl extends AbstractService implements
ContainersMonitor {
@@ -86,17 +77,11 @@ public class ContainersMonitorImpl extends AbstractService implements
private boolean pmemCheckEnabled;
private boolean vmemCheckEnabled;
private boolean containersMonitorEnabled;
-
- private boolean publishContainerMetricsToTimelineService;
private long maxVCoresAllottedForContainers;
private static final long UNKNOWN_MEMORY_LIMIT = -1L;
private int nodeCpuPercentageForYARN;
-
- // For posting entities in new timeline service in a non-blocking way
- // TODO replace with event loop in TimelineClient.
- private static ExecutorService threadPool;
@Private
public static enum ContainerMetric {
@@ -210,22 +195,6 @@ public class ContainersMonitorImpl extends AbstractService implements
1) + "). Thrashing might happen.");
}
}
-
- publishContainerMetricsToTimelineService =
- YarnConfiguration.systemMetricsPublisherEnabled(conf);
-
- if (publishContainerMetricsToTimelineService) {
- LOG.info("NodeManager has been configured to publish container " +
- "metrics to Timeline Service V2.");
- threadPool =
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
- .build());
- } else {
- LOG.warn("NodeManager has not been configured to publish container " +
- "metrics to Timeline Service V2.");
- }
-
super.serviceInit(conf);
}
@@ -269,29 +238,8 @@ public class ContainersMonitorImpl extends AbstractService implements
}
}
- shutdownAndAwaitTermination();
-
super.serviceStop();
}
-
- // TODO remove threadPool after adding non-blocking call in TimelineClient
- private static void shutdownAndAwaitTermination() {
- if (threadPool == null) {
- return;
- }
- threadPool.shutdown();
- try {
- if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
- threadPool.shutdownNow();
- if (!threadPool.awaitTermination(60, TimeUnit.SECONDS))
- LOG.error("ThreadPool did not terminate");
- }
- } catch (InterruptedException ie) {
- threadPool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
@VisibleForTesting
static class ProcessTreeInfo {
@@ -470,9 +418,6 @@ public class ContainersMonitorImpl extends AbstractService implements
ContainerId containerId = entry.getKey();
ProcessTreeInfo ptInfo = entry.getValue();
- ContainerEntity entity = new ContainerEntity();
- entity.setId(containerId.toString());
-
try {
String pId = ptInfo.getPID();
@@ -556,26 +501,6 @@ public class ContainersMonitorImpl extends AbstractService implements
containerMetricsUnregisterDelayMs).recordCpuUsage
((int)cpuUsagePercentPerCore, milliVcoresUsed);
}
-
- if (publishContainerMetricsToTimelineService) {
- // if currentPmemUsage data is available
- if (currentPmemUsage !=
- ResourceCalculatorProcessTree.UNAVAILABLE) {
- TimelineMetric memoryMetric = new TimelineMetric();
- memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
- memoryMetric.addValue(currentTime, currentPmemUsage);
- entity.addMetric(memoryMetric);
- }
- // if cpuUsageTotalCoresPercentage data is available
- if (cpuUsageTotalCoresPercentage !=
- ResourceCalculatorProcessTree.UNAVAILABLE) {
- TimelineMetric cpuMetric = new TimelineMetric();
- cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
- cpuMetric.addValue(currentTime,
- cpuUsageTotalCoresPercentage);
- entity.addMetric(cpuMetric);
- }
- }
boolean isMemoryOverLimit = false;
String msg = "";
@@ -632,23 +557,16 @@ public class ContainersMonitorImpl extends AbstractService implements
LOG.info("Removed ProcessTree with root " + pId);
}
+ ContainerImpl container =
+ (ContainerImpl) context.getContainers().get(containerId);
+ container.getNMTimelinePublisher().reportContainerResourceUsage(
+ container, currentTime, pId, currentPmemUsage,
+ cpuUsageTotalCoresPercentage);
} catch (Exception e) {
// Log the exception and proceed to the next container.
LOG.warn("Uncaught exception in ContainersMonitorImpl "
+ "while monitoring resource of " + containerId, e);
}
-
- if (publishContainerMetricsToTimelineService) {
- try {
- TimelineClient timelineClient = context.getApplications().get(
- containerId.getApplicationAttemptId().getApplicationId()).
- getTimelineClient();
- putEntityWithoutBlocking(timelineClient, entity);
- } catch (Exception e) {
- LOG.error("Exception in ContainersMonitorImpl in putting " +
- "resource usage metrics to timeline service.", e);
- }
- }
}
if (LOG.isDebugEnabled()) {
LOG.debug("Total Resource Usage stats in NM by all containers : "
@@ -671,20 +589,6 @@ public class ContainersMonitorImpl extends AbstractService implements
}
}
}
-
- private void putEntityWithoutBlocking(final TimelineClient timelineClient,
- final TimelineEntity entity) {
- Runnable publishWrapper = new Runnable() {
- public void run() {
- try {
- timelineClient.putEntities(entity);
- } catch (IOException|YarnException e) {
- LOG.error("putEntityNonBlocking get failed: " + e);
- }
- }
- };
- threadPool.execute(publishWrapper);
- }
private String formatErrorMessage(String memTypeExceeded,
long currentVmemUsage, long vmemLimit,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java
new file mode 100644
index 0000000..af8d94c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
+
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+public class NMTimelineEvent extends AbstractEvent<NMTimelineEventType> {
+ public NMTimelineEvent(NMTimelineEventType type) {
+ super(type);
+ }
+
+ public NMTimelineEvent(NMTimelineEventType type, long timestamp) {
+ super(type, timestamp);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java
new file mode 100644
index 0000000..c1129af
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
+
+public enum NMTimelineEventType {
+ // Publish the NM Timeline entity
+ TIMELINE_ENTITY_PUBLISH,
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
new file mode 100644
index 0000000..2c5c300
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -0,0 +1,376 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl.ContainerMetric;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+public class NMTimelinePublisher extends CompositeService {
+
+ private static final Log LOG = LogFactory.getLog(NMTimelinePublisher.class);
+
+ private Dispatcher dispatcher;
+ private boolean publishSystemMetrics;
+
+ private Context context;
+
+ private NodeId nodeId;
+
+ private String httpAddress;
+
+ public NMTimelinePublisher(Context context) {
+ super(NMTimelinePublisher.class.getName());
+ this.context = context;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ publishSystemMetrics =
+ YarnConfiguration.systemMetricsPublisherEnabled(conf);
+
+ if (publishSystemMetrics) {
+ dispatcher = new AsyncDispatcher();
+ dispatcher.register(NMTimelineEventType.class,
+ new ForwardingEventHandler());
+ dispatcher
+ .register(ContainerEventType.class, new ContainerEventHandler());
+ dispatcher.register(ApplicationEventType.class,
+ new ApplicationEventHandler());
+ dispatcher.register(LocalizationEventType.class,
+ new LocalizationEventDispatcher());
+ addIfService(dispatcher);
+ LOG.info("YARN system metrics publishing service is enabled");
+ } else {
+ LOG.info("YARN system metrics publishing service is not enabled");
+ }
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ // context will be updated after containerManagerImpl is started
+ // hence NMMetricsPublisher is added subservice of containerManagerImpl
+ this.nodeId = context.getNodeId();
+ this.httpAddress = nodeId.getHost() + ":" + context.getHttpPort();
+ }
+
+ protected void handleNMTimelineEvent(NMTimelineEvent event) {
+ switch (event.getType()) {
+ case TIMELINE_ENTITY_PUBLISH:
+ putEntity(((TimelinePublishEvent) event).getTimelineEntityToPublish(),
+ ((TimelinePublishEvent) event).getApplicationId());
+ break;
+ default:
+ LOG.error("Unknown NMTimelineEvent type: " + event.getType());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void reportContainerResourceUsage(Container container,
+ long createdTime, String pId, Long pmemUsage,
+ Float cpuUsageTotalCoresPercentage) {
+ if (publishSystemMetrics
+ && (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE || cpuUsageTotalCoresPercentage != ResourceCalculatorProcessTree.UNAVAILABLE)) {
+ ContainerEntity entity =
+ createContainerEntity(container.getContainerId());
+ long currentTimeMillis = System.currentTimeMillis();
+ if (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE) {
+ TimelineMetric memoryMetric = new TimelineMetric();
+ memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
+ memoryMetric.addValue(currentTimeMillis, pmemUsage);
+ entity.addMetric(memoryMetric);
+ }
+ if (cpuUsageTotalCoresPercentage != ResourceCalculatorProcessTree.UNAVAILABLE) {
+ TimelineMetric cpuMetric = new TimelineMetric();
+ cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
+ cpuMetric.addValue(currentTimeMillis, cpuUsageTotalCoresPercentage);
+ entity.addMetric(cpuMetric);
+ }
+ dispatcher.getEventHandler().handle(
+ new TimelinePublishEvent(entity, container.getContainerId()
+ .getApplicationAttemptId().getApplicationId()));
+ }
+ }
+
+ private void publishContainerCreatedEvent(ContainerEntity entity,
+ ContainerId containerId, Resource resource, Priority priority,
+ long timestamp) {
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO,
+ resource.getMemory());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO,
+ resource.getVirtualCores());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
+ nodeId.getHost());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO,
+ nodeId.getPort());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
+ priority.toString());
+ entityInfo.put(
+ ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
+ httpAddress);
+ entity.setInfo(entityInfo);
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ContainerMetricsConstants.CREATED_EVENT_TYPE);
+ tEvent.setTimestamp(timestamp);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, containerId.getApplicationAttemptId().getApplicationId());
+ }
+
+ private void publishContainerFinishedEvent(ContainerStatus containerStatus,
+ long timeStamp) {
+ ContainerId containerId = containerStatus.getContainerId();
+ TimelineEntity entity = createContainerEntity(containerId);
+
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ containerStatus.getDiagnostics());
+ eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO,
+ containerStatus.getExitStatus());
+ eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, containerStatus
+ .getState().toString());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(timeStamp);
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, containerId.getApplicationAttemptId().getApplicationId());
+ }
+
+ private static ContainerEntity createContainerEntity(ContainerId containerId) {
+ ContainerEntity entity = new ContainerEntity();
+ entity.setId(containerId.toString());
+ Identifier parentIdentifier = new Identifier();
+ parentIdentifier
+ .setType(TimelineEntityType.YARN_APPLICATION_ATTEMPT.name());
+ parentIdentifier.setId(containerId.getApplicationAttemptId().toString());
+ entity.setParent(parentIdentifier);
+ return entity;
+ }
+
+ private void putEntity(TimelineEntity entity, ApplicationId appId) {
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Publishing the entity " + entity + ", JSON-style content: "
+ + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ }
+ TimelineClient timelineClient =
+ context.getApplications().get(appId).getTimelineClient();
+ timelineClient.putEntities(entity);
+ } catch (Exception e) {
+ LOG.error("Error when publishing entity " + entity, e);
+ }
+ }
+
+ public void publishApplicationEvent(ApplicationEvent event) {
+ if (!publishSystemMetrics) {
+ return;
+ }
+ // publish only when the desired event is received
+ switch (event.getType()) {
+ case INIT_APPLICATION:
+ case FINISH_APPLICATION:
+ case APPLICATION_CONTAINER_FINISHED:
+ case APPLICATION_LOG_HANDLING_FAILED:
+ dispatcher.getEventHandler().handle(event);
+ break;
+
+ default:
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(event.getType()
+ + " is not a desired ApplicationEvent which needs to be published by"
+ + " NMTimelinePublisher");
+ }
+ break;
+ }
+ }
+
+ public void publishContainerEvent(ContainerEvent event) {
+ if (!publishSystemMetrics) {
+ return;
+ }
+ // publish only when the desired event is received
+ switch (event.getType()) {
+ case INIT_CONTAINER:
+ dispatcher.getEventHandler().handle(event);
+ break;
+
+ default:
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(event.getType()
+ + " is not a desired ContainerEvent which needs to be published by"
+ + " NMTimelinePublisher");
+ }
+ break;
+ }
+ }
+
+ public void publishLocalizationEvent(LocalizationEvent event) {
+ if (!publishSystemMetrics) {
+ return;
+ }
+ // publish only when the desired event is received
+ switch (event.getType()) {
+ case CONTAINER_RESOURCES_LOCALIZED:
+ case INIT_CONTAINER_RESOURCES:
+ dispatcher.getEventHandler().handle(event);
+ break;
+
+ default:
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(event.getType()
+ + " is not a desired LocalizationEvent which needs to be published"
+ + " by NMTimelinePublisher");
+ }
+ break;
+ }
+ }
+
+ private class ApplicationEventHandler implements
+ EventHandler<ApplicationEvent> {
+ @Override
+ public void handle(ApplicationEvent event) {
+ switch (event.getType()) {
+ case APPLICATION_CONTAINER_FINISHED:
+ // this is actually used to publish the container Event
+ ApplicationContainerFinishedEvent evnt =
+ (ApplicationContainerFinishedEvent) event;
+ publishContainerFinishedEvent(evnt.getContainerStatus(),
+ event.getTimestamp());
+ break;
+ default:
+ LOG.error("Seems like event type is captured only in "
+ + "publishApplicationEvent method and not handled here");
+ break;
+ }
+ }
+ }
+
+ private class ContainerEventHandler implements EventHandler<ContainerEvent> {
+ @Override
+ public void handle(ContainerEvent event) {
+ ContainerId containerId = event.getContainerID();
+ Container container = context.getContainers().get(containerId);
+ long timestamp = event.getTimestamp();
+ ContainerEntity entity = createContainerEntity(containerId);
+
+ switch (event.getType()) {
+ case INIT_CONTAINER:
+ publishContainerCreatedEvent(entity, containerId,
+ container.getResource(), container.getPriority(), timestamp);
+ break;
+ default:
+ LOG.error("Seems like event type is captured only in "
+ + "publishContainerEvent method and not handled here");
+ break;
+ }
+ }
+ }
+
+ private static final class LocalizationEventDispatcher implements
+ EventHandler<LocalizationEvent> {
+ @Override
+ public void handle(LocalizationEvent event) {
+ switch (event.getType()) {
+ case INIT_CONTAINER_RESOURCES:
+ case CONTAINER_RESOURCES_LOCALIZED:
+ // TODO after priority based flush jira is finished
+ break;
+ default:
+ LOG.error("Seems like event type is captured only in "
+ + "publishLocalizationEvent method and not handled here");
+ break;
+ }
+ }
+ }
+
+ /**
+ * EventHandler implementation which forward events to NMMetricsPublisher.
+ * Making use of it, NMMetricsPublisher can avoid to have a public handle
+ * method.
+ */
+ private final class ForwardingEventHandler implements
+ EventHandler<NMTimelineEvent> {
+
+ @Override
+ public void handle(NMTimelineEvent event) {
+ handleNMTimelineEvent(event);
+ }
+ }
+
+ private static class TimelinePublishEvent extends NMTimelineEvent {
+ private ApplicationId appId;
+ private TimelineEntity entityToPublish;
+
+ public TimelinePublishEvent(TimelineEntity entity, ApplicationId appId) {
+ super(NMTimelineEventType.TIMELINE_ENTITY_PUBLISH, System
+ .currentTimeMillis());
+ this.appId = appId;
+ this.entityToPublish = entity;
+ }
+
+ public ApplicationId getApplicationId() {
+ return appId;
+ }
+
+ public TimelineEntity getTimelineEntityToPublish() {
+ return entityToPublish;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index 47f6df5..98bb827 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.ContainerContext;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
@@ -250,8 +251,10 @@ public class TestNodeStatusUpdater {
firstContainerID, InetAddress.getByName("localhost")
.getCanonicalHostName(), 1234, user, resource,
currentTime + 10000, 123, "password".getBytes(), currentTime));
- Container container = new ContainerImpl(conf, mockDispatcher,
- stateStore, launchContext, null, mockMetrics, containerToken);
+ Container container =
+ new ContainerImpl(conf, mockDispatcher, stateStore, launchContext,
+ null, mockMetrics, containerToken,
+ mock(Context.class));
this.context.getContainers().put(firstContainerID, container);
} else if (heartBeatID == 2) {
// Checks on the RM end
@@ -289,8 +292,10 @@ public class TestNodeStatusUpdater {
secondContainerID, InetAddress.getByName("localhost")
.getCanonicalHostName(), 1234, user, resource,
currentTime + 10000, 123, "password".getBytes(), currentTime));
- Container container = new ContainerImpl(conf, mockDispatcher,
- stateStore, launchContext, null, mockMetrics, containerToken);
+ Container container =
+ new ContainerImpl(conf, mockDispatcher, stateStore, launchContext,
+ null, mockMetrics, containerToken,
+ mock(Context.class));
this.context.getContainers().put(secondContainerID, container);
} else if (heartBeatID == 3) {
// Checks on the RM end
@@ -977,7 +982,8 @@ public class TestNodeStatusUpdater {
"password".getBytes(), 0);
Container anyCompletedContainer = new ContainerImpl(conf, null,
null, null, null, null,
- BuilderUtils.newContainerTokenIdentifier(containerToken)) {
+ BuilderUtils.newContainerTokenIdentifier(containerToken),
+ mock(Context.class)) {
@Override
public ContainerState getCurrentState() {
@@ -998,7 +1004,7 @@ public class TestNodeStatusUpdater {
"password".getBytes(), 0);
Container runningContainer =
new ContainerImpl(conf, null, null, null, null, null,
- BuilderUtils.newContainerTokenIdentifier(runningContainerToken)) {
+ BuilderUtils.newContainerTokenIdentifier(runningContainerToken), mock(Context.class)) {
@Override
public ContainerState getCurrentState() {
return ContainerState.RUNNING;
@@ -1054,9 +1060,10 @@ public class TestNodeStatusUpdater {
BuilderUtils.newContainerToken(containerId, "host", 1234, "user",
BuilderUtils.newResource(1024, 1), 0, 123,
"password".getBytes(), 0);
+
Container completedContainer = new ContainerImpl(conf, null,
null, null, null, null,
- BuilderUtils.newContainerTokenIdentifier(containerToken)) {
+ BuilderUtils.newContainerTokenIdentifier(containerToken),mock(Context.class)) {
@Override
public ContainerState getCurrentState() {
return ContainerState.COMPLETE;
@@ -1093,7 +1100,8 @@ public class TestNodeStatusUpdater {
"password".getBytes(), 0);
Container anyCompletedContainer = new ContainerImpl(conf, null,
null, null, null, null,
- BuilderUtils.newContainerTokenIdentifier(containerToken)) {
+ BuilderUtils.newContainerTokenIdentifier(containerToken),
+ mock(Context.class)) {
@Override
public ContainerState getCurrentState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index bf4fd52..35f2d39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
@@ -688,5 +689,13 @@ public abstract class BaseAMRMProxyTest {
return null;
}
+ @Override
+ public void setNMTimelinePublisher(NMTimelinePublisher nmMetricsPublisher) {
+ }
+
+ @Override
+ public NMTimelinePublisher getNMTimelinePublisher() {
+ return null;
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
index 1380752..8c4bd25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
@@ -53,6 +54,7 @@ import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
import org.apache.hadoop.yarn.server.api.AuxiliaryService;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.junit.Assert;
@@ -193,7 +195,7 @@ public class TestAuxServices {
ContainerId.newContainerId(attemptId, 1), "", "",
Resource.newInstance(1, 1), 0,0,0, Priority.newInstance(0), 0);
Container container = new ContainerImpl(null, null, null, null, null,
- null, cti);
+ null, cti, mock(Context.class));
ContainerId containerId = container.getContainerId();
Resource resource = container.getResource();
event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_INIT,container);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index 31f4702..8b1ef67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -100,10 +100,12 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreServic
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestContainerManagerRecovery extends BaseContainerManagerTest {
@@ -633,6 +635,12 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
boolean blockNewContainerRequests) {
// do nothing
}
+
+ @Override
+ public NMTimelinePublisher createNMTimelinePublisher(Context context) {
+ NMTimelinePublisher timelinePublisher = mock(NMTimelinePublisher.class);
+ return timelinePublisher;
+ }
};
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
index 002d4cf..38b3172f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
@@ -38,7 +38,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
@@ -579,7 +582,7 @@ public class TestApplication {
public void containerFinished(int containerNum) {
app.handle(new ApplicationContainerFinishedEvent(containers.get(
- containerNum).getContainerId()));
+ containerNum).cloneAndGetContainerStatus()));
drainDispatcherEvents();
}
@@ -616,6 +619,8 @@ public class TestApplication {
when(c.getLaunchContext()).thenReturn(launchContext);
when(launchContext.getApplicationACLs()).thenReturn(
new HashMap<ApplicationAccessType, String>());
+ when(c.cloneAndGetContainerStatus()).thenReturn(BuilderUtils.newContainerStatus(cId,
+ ContainerState.NEW, "", 0, Resource.newInstance(1024, 1)));
return c;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index 2834e30..aa22e5e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -808,7 +808,7 @@ public class TestContainer {
when(ctxt.getServiceData()).thenReturn(serviceData);
c = new ContainerImpl(conf, dispatcher, new NMNullStateStoreService(),
- ctxt, null, metrics, identifier);
+ ctxt, null, metrics, identifier, mock(Context.class));
dispatcher.register(ContainerEventType.class,
new EventHandler<ContainerEvent>() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 394a92c..37c726f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -144,4 +145,9 @@ public class MockContainer implements Container {
public NMContainerStatus getNMContainerStatus() {
return null;
}
+
+ @Override
+ public Priority getPriority() {
+ return Priority.UNDEFINED;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
index ed94fb6..e839f47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
@@ -220,7 +220,8 @@ public class TestNMWebServer {
Container container =
new ContainerImpl(conf, dispatcher, stateStore, launchContext,
null, metrics,
- BuilderUtils.newContainerTokenIdentifier(containerToken)) {
+ BuilderUtils.newContainerTokenIdentifier(containerToken),
+ mock(Context.class)) {
@Override
public ContainerState getContainerState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index befaa83..4147d42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -137,9 +137,19 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
// intercept the event of the AM container being stopped and remove the app
// level collector service
if (context.getContainerType() == ContainerType.APPLICATION_MASTER) {
- ApplicationId appId = context.getContainerId().
- getApplicationAttemptId().getApplicationId();
- removeApplication(appId);
+ final ApplicationId appId =
+ context.getContainerId().getApplicationAttemptId().getApplicationId();
+ new Thread(new Runnable() {
+ public void run() {
+ try {
+ // TODO Temporary Fix until solution for YARN-3995 is finalized.
+ Thread.sleep(1000l);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ removeApplication(appId);
+ }
+ }).start();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fb80b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index 7cc612d..dafc76e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -98,6 +98,15 @@ public class TestPerNodeTimelineCollectorsAuxService {
when(context.getContainerType()).thenReturn(
ContainerType.APPLICATION_MASTER);
auxService.stopContainer(context);
+
+ // TODO Temporary Fix until solution for YARN-3995 is finalized
+ for (int i = 0; i < 4; i++) {
+ Thread.sleep(500l);
+ if (!auxService.hasApplication(appAttemptId.getApplicationId())) {
+ break;
+ }
+ }
+
// auxService should not have that app
assertFalse(auxService.hasApplication(appAttemptId.getApplicationId()));
auxService.close();
[43/50] [abbrv] hadoop git commit: YARN-4074. [timeline reader]
implement support for querying for flows and flow runs (sjlee via vrushali)
Posted by sj...@apache.org.
YARN-4074. [timeline reader] implement support for querying for flows and flow runs (sjlee via vrushali)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4d8d8fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4d8d8fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4d8d8fd
Branch: refs/heads/YARN-2928-rebase
Commit: c4d8d8fd16fca41aab2a8d5ae50ddc34a4967c3f
Parents: 78e76c7
Author: Vrushali <vr...@apache.org>
Authored: Tue Sep 22 13:42:30 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 17:05:04 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../timelineservice/FlowActivityEntity.java | 183 ++++++++
.../api/records/timelineservice/FlowEntity.java | 103 -----
.../records/timelineservice/FlowRunEntity.java | 121 ++++++
.../timelineservice/TimelineEntityType.java | 31 +-
.../TestTimelineServiceRecords.java | 14 +-
.../TestTimelineServiceClientIntegration.java | 2 +-
.../collector/TimelineCollectorWebService.java | 6 +-
.../storage/ApplicationEntityReader.java | 229 ++++++++++
.../storage/FlowActivityEntityReader.java | 168 +++++++
.../storage/FlowRunEntityReader.java | 136 ++++++
.../storage/GenericEntityReader.java | 389 +++++++++++++++++
.../storage/HBaseTimelineReaderImpl.java | 434 +------------------
.../storage/TimelineEntityReader.java | 223 ++++++++++
.../storage/TimelineEntityReaderFactory.java | 97 +++++
.../storage/application/ApplicationRowKey.java | 68 ++-
.../storage/apptoflow/AppToFlowRowKey.java | 31 ++
.../storage/common/BaseTable.java | 3 +-
.../storage/entity/EntityRowKey.java | 76 +++-
.../storage/flow/FlowActivityRowKey.java | 7 +-
.../storage/flow/FlowRunRowKey.java | 50 ++-
.../storage/flow/FlowScanner.java | 18 +-
.../storage/TestHBaseTimelineStorage.java | 34 +-
.../storage/flow/TestFlowDataGenerator.java | 39 +-
.../flow/TestHBaseStorageFlowActivity.java | 131 ++++--
.../storage/flow/TestHBaseStorageFlowRun.java | 105 +++--
26 files changed, 2021 insertions(+), 680 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ecbe9de..3d2de5c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -106,6 +106,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3901. Populate flow run data in the flow_run & flow activity tables
(Vrushali C via sjlee)
+ YARN-4074. [timeline reader] implement support for querying for flows
+ and flow runs (sjlee via vrushali)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowActivityEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowActivityEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowActivityEntity.java
new file mode 100644
index 0000000..163bd5c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowActivityEntity.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timelineservice;
+
+import java.util.Collection;
+import java.util.Date;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+
+import javax.xml.bind.annotation.XmlElement;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * Entity that represents a record for flow activity. It's essentially a
+ * container entity for flow runs with limited information.
+ */
+@Public
+@Unstable
+public class FlowActivityEntity extends TimelineEntity {
+ public static final String CLUSTER_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "CLUSTER";
+ public static final String DATE_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "DATE";
+ public static final String USER_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "USER";
+ public static final String FLOW_NAME_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_NAME";
+
+ private final NavigableSet<FlowRunEntity> flowRuns = new TreeSet<>();
+
+ public FlowActivityEntity() {
+ super(TimelineEntityType.YARN_FLOW_ACTIVITY.toString());
+ // set config to null
+ setConfigs(null);
+ }
+
+ public FlowActivityEntity(String cluster, long time, String user,
+ String flowName) {
+ this();
+ setCluster(cluster);
+ setDate(time);
+ setUser(user);
+ setFlowName(flowName);
+ }
+
+ public FlowActivityEntity(TimelineEntity entity) {
+ super(entity);
+ if (!TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entity.getType())) {
+ throw new IllegalArgumentException("Incompatible entity type: " +
+ getId());
+ }
+ // set config to null
+ setConfigs(null);
+ }
+
+ @XmlElement(name = "id")
+ @Override
+ public String getId() {
+ // flow activity: cluster/day/user@flow_name
+ String id = super.getId();
+ if (id == null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getCluster());
+ sb.append('/');
+ sb.append(getDate().getTime());
+ sb.append('/');
+ sb.append(getUser());
+ sb.append('@');
+ sb.append(getFlowName());
+ id = sb.toString();
+ setId(id);
+ }
+ return id;
+ }
+
+ @Override
+ public int compareTo(TimelineEntity entity) {
+ int comparison = getType().compareTo(entity.getType());
+ if (comparison == 0) {
+ // order by cluster, date (descending), user, and flow name
+ FlowActivityEntity other = (FlowActivityEntity)entity;
+ int clusterComparison = getCluster().compareTo(other.getCluster());
+ if (clusterComparison != 0) {
+ return clusterComparison;
+ }
+ int dateComparisonDescending =
+ (int)(other.getDate().getTime() - getDate().getTime()); // descending
+ if (dateComparisonDescending != 0) {
+ return dateComparisonDescending; // descending
+ }
+ int userComparison = getUser().compareTo(other.getUser());
+ if (userComparison != 0) {
+ return userComparison;
+ }
+ return getFlowName().compareTo(other.getFlowName());
+ } else {
+ return comparison;
+ }
+ }
+
+ /**
+ * Reuse the base class equals method.
+ */
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj);
+ }
+
+ /**
+ * Reuse the base class hashCode method.
+ */
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ public String getCluster() {
+ return (String)getInfo().get(CLUSTER_INFO_KEY);
+ }
+
+ public void setCluster(String cluster) {
+ addInfo(CLUSTER_INFO_KEY, cluster);
+ }
+
+ public Date getDate() {
+ return (Date)getInfo().get(DATE_INFO_KEY);
+ }
+
+ public void setDate(long time) {
+ Date date = new Date(time);
+ addInfo(DATE_INFO_KEY, date);
+ }
+
+ public String getUser() {
+ return (String)getInfo().get(USER_INFO_KEY);
+ }
+
+ public void setUser(String user) {
+ addInfo(USER_INFO_KEY, user);
+ }
+
+ public String getFlowName() {
+ return (String)getInfo().get(FLOW_NAME_INFO_KEY);
+ }
+
+ public void setFlowName(String flowName) {
+ addInfo(FLOW_NAME_INFO_KEY, flowName);
+ }
+
+ public void addFlowRun(FlowRunEntity run) {
+ flowRuns.add(run);
+ }
+
+ public void addFlowRuns(Collection<FlowRunEntity> runs) {
+ flowRuns.addAll(runs);
+ }
+
+ @XmlElement(name = "flowruns")
+ public NavigableSet<FlowRunEntity> getFlowRuns() {
+ return flowRuns;
+ }
+
+ public int getNumberOfRuns() {
+ return flowRuns.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java
deleted file mode 100644
index 4554778..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.api.records.timelineservice;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import javax.xml.bind.annotation.XmlElement;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class FlowEntity extends HierarchicalTimelineEntity {
- public static final String USER_INFO_KEY =
- TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "USER";
- public static final String FLOW_NAME_INFO_KEY =
- TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_NAME";
- public static final String FLOW_VERSION_INFO_KEY =
- TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_VERSION";
- public static final String FLOW_RUN_ID_INFO_KEY =
- TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_RUN_ID";
-
- public FlowEntity() {
- super(TimelineEntityType.YARN_FLOW.toString());
- }
-
- public FlowEntity(TimelineEntity entity) {
- super(entity);
- if (!entity.getType().equals(TimelineEntityType.YARN_FLOW.toString())) {
- throw new IllegalArgumentException("Incompatible entity type: " + getId());
- }
- }
-
- @XmlElement(name = "id")
- @Override
- public String getId() {
- //Flow id schema: user@flow_name(or id)/version/run_id
- String id = super.getId();
- if (id == null) {
- StringBuilder sb = new StringBuilder();
- sb.append(getInfo().get(USER_INFO_KEY).toString());
- sb.append('@');
- sb.append(getInfo().get(FLOW_NAME_INFO_KEY).toString());
- sb.append('/');
- sb.append(getInfo().get(FLOW_VERSION_INFO_KEY).toString());
- sb.append('/');
- sb.append(getInfo().get(FLOW_RUN_ID_INFO_KEY).toString());
- id = sb.toString();
- setId(id);
- }
- return id;
- }
-
- public String getUser() {
- Object user = getInfo().get(USER_INFO_KEY);
- return user == null ? null : user.toString();
- }
-
- public void setUser(String user) {
- addInfo(USER_INFO_KEY, user);
- }
-
- public String getName() {
- Object name = getInfo().get(FLOW_NAME_INFO_KEY);
- return name == null ? null : name.toString();
- }
-
- public void setName(String name) {
- addInfo(FLOW_NAME_INFO_KEY, name);
- }
-
- public String getVersion() {
- Object version = getInfo().get(FLOW_VERSION_INFO_KEY);
- return version == null ? null : version.toString();
- }
-
- public void setVersion(String version) {
- addInfo(FLOW_VERSION_INFO_KEY, version);
- }
-
- public long getRunId() {
- Object runId = getInfo().get(FLOW_RUN_ID_INFO_KEY);
- return runId == null ? 0L : (Long) runId;
- }
-
- public void setRunId(long runId) {
- addInfo(FLOW_RUN_ID_INFO_KEY, runId);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
new file mode 100644
index 0000000..3c3ffb4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timelineservice;
+
+import javax.xml.bind.annotation.XmlElement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class FlowRunEntity extends HierarchicalTimelineEntity {
+ public static final String USER_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "USER";
+ public static final String FLOW_NAME_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_NAME";
+ public static final String FLOW_VERSION_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_VERSION";
+ public static final String FLOW_RUN_ID_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_RUN_ID";
+ public static final String FLOW_RUN_END_TIME =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_RUN_END_TIME";
+
+ public FlowRunEntity() {
+ super(TimelineEntityType.YARN_FLOW_RUN.toString());
+ // set config to null
+ setConfigs(null);
+ }
+
+ public FlowRunEntity(TimelineEntity entity) {
+ super(entity);
+ if (!entity.getType().equals(TimelineEntityType.YARN_FLOW_RUN.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ }
+ // set config to null
+ setConfigs(null);
+ }
+
+ @XmlElement(name = "id")
+ @Override
+ public String getId() {
+ //Flow id schema: user@flow_name(or id)/run_id
+ String id = super.getId();
+ if (id == null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getInfo().get(USER_INFO_KEY).toString());
+ sb.append('@');
+ sb.append(getInfo().get(FLOW_NAME_INFO_KEY).toString());
+ sb.append('/');
+ sb.append(getInfo().get(FLOW_RUN_ID_INFO_KEY).toString());
+ id = sb.toString();
+ setId(id);
+ }
+ return id;
+ }
+
+ public String getUser() {
+ return (String)getInfo().get(USER_INFO_KEY);
+ }
+
+ public void setUser(String user) {
+ addInfo(USER_INFO_KEY, user);
+ }
+
+ public String getName() {
+ return (String)getInfo().get(FLOW_NAME_INFO_KEY);
+ }
+
+ public void setName(String name) {
+ addInfo(FLOW_NAME_INFO_KEY, name);
+ }
+
+ public String getVersion() {
+ return (String)getInfo().get(FLOW_VERSION_INFO_KEY);
+ }
+
+ public void setVersion(String version) {
+ addInfo(FLOW_VERSION_INFO_KEY, version);
+ }
+
+ public long getRunId() {
+ Object runId = getInfo().get(FLOW_RUN_ID_INFO_KEY);
+ return runId == null ? 0L : (Long) runId;
+ }
+
+ public void setRunId(long runId) {
+ addInfo(FLOW_RUN_ID_INFO_KEY, runId);
+ }
+
+ public long getStartTime() {
+ return getCreatedTime();
+ }
+
+ public void setStartTime(long startTime) {
+ setCreatedTime(startTime);
+ }
+
+ public long getMaxEndTime() {
+ Object time = getInfo().get(FLOW_RUN_END_TIME);
+ return time == null ? 0L : (Long)time;
+ }
+
+ public void setMaxEndTime(long endTime) {
+ addInfo(FLOW_RUN_END_TIME, endTime);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java
index 6062fe1..ba32e20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java
@@ -24,21 +24,25 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Unstable
public enum TimelineEntityType {
YARN_CLUSTER,
- YARN_FLOW,
+ YARN_FLOW_RUN,
YARN_APPLICATION,
YARN_APPLICATION_ATTEMPT,
YARN_CONTAINER,
YARN_USER,
- YARN_QUEUE;
+ YARN_QUEUE,
+ YARN_FLOW_ACTIVITY;
+ /**
+ * Whether the input type can be a parent of this entity.
+ */
public boolean isParent(TimelineEntityType type) {
switch (this) {
case YARN_CLUSTER:
return false;
- case YARN_FLOW:
- return YARN_FLOW == type || YARN_CLUSTER == type;
+ case YARN_FLOW_RUN:
+ return YARN_FLOW_RUN == type || YARN_CLUSTER == type;
case YARN_APPLICATION:
- return YARN_FLOW == type || YARN_CLUSTER == type;
+ return YARN_FLOW_RUN == type || YARN_CLUSTER == type;
case YARN_APPLICATION_ATTEMPT:
return YARN_APPLICATION == type;
case YARN_CONTAINER:
@@ -50,12 +54,15 @@ public enum TimelineEntityType {
}
}
+ /**
+ * Whether the input type can be a child of this entity.
+ */
public boolean isChild(TimelineEntityType type) {
switch (this) {
case YARN_CLUSTER:
- return YARN_FLOW == type || YARN_APPLICATION == type;
- case YARN_FLOW:
- return YARN_FLOW == type || YARN_APPLICATION == type;
+ return YARN_FLOW_RUN == type || YARN_APPLICATION == type;
+ case YARN_FLOW_RUN:
+ return YARN_FLOW_RUN == type || YARN_APPLICATION == type;
case YARN_APPLICATION:
return YARN_APPLICATION_ATTEMPT == type;
case YARN_APPLICATION_ATTEMPT:
@@ -68,4 +75,12 @@ public enum TimelineEntityType {
return false;
}
}
+
+ /**
+ * Whether the type of this entity matches the type indicated by the input
+ * argument.
+ */
+ public boolean matches(String typeString) {
+ return toString().equals(typeString);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index 78943e0..7c9acf2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -182,14 +182,14 @@ public class TestTimelineServiceRecords {
ClusterEntity cluster = new ClusterEntity();
cluster.setId("test cluster id");
- FlowEntity flow1 = new FlowEntity();
+ FlowRunEntity flow1 = new FlowRunEntity();
//flow1.setId("test flow id 1");
flow1.setUser(user.getId());
flow1.setName("test flow name 1");
flow1.setVersion("test flow version 1");
flow1.setRunId(1L);
- FlowEntity flow2 = new FlowEntity();
+ FlowRunEntity flow2 = new FlowRunEntity();
//flow2.setId("test flow run id 2");
flow2.setUser(user.getId());
flow2.setName("test flow name 2");
@@ -213,19 +213,19 @@ public class TestTimelineServiceRecords {
ApplicationAttemptId.newInstance(
ApplicationId.newInstance(0, 1), 1), 1).toString());
- cluster.addChild(TimelineEntityType.YARN_FLOW.toString(), flow1.getId());
+ cluster.addChild(TimelineEntityType.YARN_FLOW_RUN.toString(), flow1.getId());
flow1
.setParent(TimelineEntityType.YARN_CLUSTER.toString(), cluster.getId());
- flow1.addChild(TimelineEntityType.YARN_FLOW.toString(), flow2.getId());
- flow2.setParent(TimelineEntityType.YARN_FLOW.toString(), flow1.getId());
+ flow1.addChild(TimelineEntityType.YARN_FLOW_RUN.toString(), flow2.getId());
+ flow2.setParent(TimelineEntityType.YARN_FLOW_RUN.toString(), flow1.getId());
flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(), app1.getId());
flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(), app2.getId());
- app1.setParent(TimelineEntityType.YARN_FLOW.toString(), flow2.getId());
+ app1.setParent(TimelineEntityType.YARN_FLOW_RUN.toString(), flow2.getId());
app1.addChild(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
appAttempt.getId());
appAttempt
.setParent(TimelineEntityType.YARN_APPLICATION.toString(), app1.getId());
- app2.setParent(TimelineEntityType.YARN_FLOW.toString(), flow2.getId());
+ app2.setParent(TimelineEntityType.YARN_FLOW_RUN.toString(), flow2.getId());
appAttempt.addChild(TimelineEntityType.YARN_CONTAINER.toString(),
container.getId());
container.setParent(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 69031a2..5672759 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -107,7 +107,7 @@ public class TestTimelineServiceClientIntegration {
client.start();
ClusterEntity cluster = new ClusterEntity();
cluster.setId(YarnConfiguration.DEFAULT_RM_CLUSTER_ID);
- FlowEntity flow = new FlowEntity();
+ FlowRunEntity flow = new FlowRunEntity();
flow.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
flow.setName("test_flow_name");
flow.setVersion("test_flow_version");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
index 42fa365..8f595e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
@@ -47,7 +47,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEnti
import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -205,8 +205,8 @@ public class TimelineCollectorWebService {
case YARN_CLUSTER:
entitiesToReturn.addEntity(new ClusterEntity(entity));
break;
- case YARN_FLOW:
- entitiesToReturn.addEntity(new FlowEntity(entity));
+ case YARN_FLOW_RUN:
+ entitiesToReturn.addEntity(new FlowRunEntity(entity));
break;
case YARN_APPLICATION:
entitiesToReturn.addEntity(new ApplicationEntity(entity));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
new file mode 100644
index 0000000..dfbc31d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
+
+/**
+ * Timeline entity reader for application entities that are stored in the
+ * application table.
+ */
+class ApplicationEntityReader extends GenericEntityReader {
+ private static final ApplicationTable APPLICATION_TABLE =
+ new ApplicationTable();
+
+ public ApplicationEntityReader(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+ eventFilters, fieldsToRetrieve);
+ }
+
+ public ApplicationEntityReader(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowId, flowRunId, appId, entityType, entityId,
+ fieldsToRetrieve);
+ }
+
+ /**
+ * Uses the {@link ApplicationTable}.
+ */
+ protected BaseTable<?> getTable() {
+ return APPLICATION_TABLE;
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ byte[] rowKey =
+ ApplicationRowKey.getRowKey(clusterId, userId, flowId, flowRunId,
+ appId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ return table.getResult(hbaseConf, conn, get);
+ }
+
+ @Override
+ protected Iterable<Result> getResults(Configuration hbaseConf,
+ Connection conn) throws IOException {
+ // If getEntities() is called for an application, there can be at most
+ // one entity. If the entity passes the filter, it is returned. Otherwise,
+ // an empty set is returned.
+ byte[] rowKey = ApplicationRowKey.getRowKey(clusterId, userId, flowId,
+ flowRunId, appId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Result result = table.getResult(hbaseConf, conn, get);
+ TimelineEntity entity = parseEntity(result);
+ Set<Result> set;
+ if (entity != null) {
+ set = Collections.singleton(result);
+ } else {
+ set = Collections.emptySet();
+ }
+ return set;
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ TimelineEntity entity = new TimelineEntity();
+ entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
+ String entityId = ApplicationColumn.ID.readResult(result).toString();
+ entity.setId(entityId);
+
+ // fetch created time
+ Number createdTime =
+ (Number)ApplicationColumn.CREATED_TIME.readResult(result);
+ entity.setCreatedTime(createdTime.longValue());
+ if (!singleEntityRead && (entity.getCreatedTime() < createdTimeBegin ||
+ entity.getCreatedTime() > createdTimeEnd)) {
+ return null;
+ }
+
+ // fetch modified time
+ Number modifiedTime =
+ (Number)ApplicationColumn.MODIFIED_TIME.readResult(result);
+ entity.setModifiedTime(modifiedTime.longValue());
+ if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin ||
+ entity.getModifiedTime() > modifiedTimeEnd)) {
+ return null;
+ }
+
+ // fetch is related to entities
+ boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
+ readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
+ true);
+ if (checkIsRelatedTo && !TimelineReaderUtils.matchRelations(
+ entity.getIsRelatedToEntities(), isRelatedTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
+ entity.getIsRelatedToEntities().clear();
+ }
+ }
+
+ // fetch relates to entities
+ boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
+ readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
+ false);
+ if (checkRelatesTo && !TimelineReaderUtils.matchRelations(
+ entity.getRelatesToEntities(), relatesTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.RELATES_TO)) {
+ entity.getRelatesToEntities().clear();
+ }
+ }
+
+ // fetch info
+ boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
+ readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
+ if (checkInfo &&
+ !TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.INFO)) {
+ entity.getInfo().clear();
+ }
+ }
+
+ // fetch configs
+ boolean checkConfigs = configFilters != null && configFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
+ readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
+ if (checkConfigs && !TimelineReaderUtils.matchFilters(
+ entity.getConfigs(), configFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.CONFIGS)) {
+ entity.getConfigs().clear();
+ }
+ }
+
+ // fetch events
+ boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
+ readEvents(entity, result, true);
+ if (checkEvents && !TimelineReaderUtils.matchEventFilters(
+ entity.getEvents(), eventFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.EVENTS)) {
+ entity.getEvents().clear();
+ }
+ }
+
+ // fetch metrics
+ boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
+ readMetrics(entity, result, ApplicationColumnPrefix.METRIC);
+ if (checkMetrics && !TimelineReaderUtils.matchMetricFilters(
+ entity.getMetrics(), metricFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.METRICS)) {
+ entity.getMetrics().clear();
+ }
+ }
+ return entity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
new file mode 100644
index 0000000..d5ece2e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for flow activity entities that are stored in the
+ * flow activity table.
+ */
+class FlowActivityEntityReader extends TimelineEntityReader {
+ private static final FlowActivityTable FLOW_ACTIVITY_TABLE =
+ new FlowActivityTable();
+
+ public FlowActivityEntityReader(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+ eventFilters, fieldsToRetrieve);
+ }
+
+ public FlowActivityEntityReader(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowId, flowRunId, appId, entityType, entityId,
+ fieldsToRetrieve);
+ }
+
+ /**
+ * Uses the {@link FlowActivityTable}.
+ */
+ @Override
+ protected BaseTable<?> getTable() {
+ return FLOW_ACTIVITY_TABLE;
+ }
+
+ /**
+ * Since this is strictly sorted by the row key, it is sufficient to collect
+ * the first results as specified by the limit.
+ */
+ @Override
+ public Set<TimelineEntity> readEntities(Configuration hbaseConf,
+ Connection conn) throws IOException {
+ validateParams();
+ augmentParams(hbaseConf, conn);
+
+ NavigableSet<TimelineEntity> entities = new TreeSet<>();
+ Iterable<Result> results = getResults(hbaseConf, conn);
+ for (Result result : results) {
+ TimelineEntity entity = parseEntity(result);
+ if (entity == null) {
+ continue;
+ }
+ entities.add(entity);
+ if (entities.size() == limit) {
+ break;
+ }
+ }
+ return entities;
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ if (limit == null || limit < 0) {
+ limit = TimelineReader.DEFAULT_LIMIT;
+ }
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ throw new UnsupportedOperationException(
+ "we don't support a single entity query");
+ }
+
+ @Override
+ protected Iterable<Result> getResults(Configuration hbaseConf,
+ Connection conn) throws IOException {
+ Scan scan = new Scan();
+ scan.setRowPrefixFilter(FlowActivityRowKey.getRowKeyPrefix(clusterId));
+ // use the page filter to limit the result to the page size
+ // the scanner may still return more than the limit; therefore we need to
+ // read the right number as we iterate
+ scan.setFilter(new PageFilter(limit));
+ return table.getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ FlowActivityRowKey rowKey = FlowActivityRowKey.parseRowKey(result.getRow());
+
+ long time = rowKey.getDayTimestamp();
+ String user = rowKey.getUserId();
+ String flowName = rowKey.getFlowId();
+
+ FlowActivityEntity flowActivity =
+ new FlowActivityEntity(clusterId, time, user, flowName);
+ // set the id
+ flowActivity.setId(flowActivity.getId());
+ // get the list of run ids along with the version that are associated with
+ // this flow on this day
+ Map<String, Object> runIdsMap =
+ FlowActivityColumnPrefix.RUN_ID.readResults(result);
+ for (Map.Entry<String, Object> e : runIdsMap.entrySet()) {
+ Long runId = Long.valueOf(e.getKey());
+ String version = (String)e.getValue();
+ FlowRunEntity flowRun = new FlowRunEntity();
+ flowRun.setUser(user);
+ flowRun.setName(flowName);
+ flowRun.setRunId(runId);
+ flowRun.setVersion(version);
+ // set the id
+ flowRun.setId(flowRun.getId());
+ flowActivity.addFlowRun(flowRun);
+ }
+
+ return flowActivity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
new file mode 100644
index 0000000..ced795d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for flow run entities that are stored in the flow run
+ * table.
+ */
+class FlowRunEntityReader extends TimelineEntityReader {
+ private static final FlowRunTable FLOW_RUN_TABLE = new FlowRunTable();
+
+ public FlowRunEntityReader(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+ eventFilters, fieldsToRetrieve);
+ }
+
+ public FlowRunEntityReader(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowId, flowRunId, appId, entityType, entityId,
+ fieldsToRetrieve);
+ }
+
+ /**
+ * Uses the {@link FlowRunTable}.
+ */
+ @Override
+ protected BaseTable<?> getTable() {
+ return FLOW_RUN_TABLE;
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ Preconditions.checkNotNull(userId, "userId shouldn't be null");
+ Preconditions.checkNotNull(flowId, "flowId shouldn't be null");
+ Preconditions.checkNotNull(flowRunId, "flowRunId shouldn't be null");
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn) {
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ byte[] rowKey =
+ FlowRunRowKey.getRowKey(clusterId, userId, flowId, flowRunId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ return table.getResult(hbaseConf, conn, get);
+ }
+
+ @Override
+ protected Iterable<Result> getResults(Configuration hbaseConf,
+ Connection conn) throws IOException {
+ throw new UnsupportedOperationException(
+ "multiple entity query is not supported");
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ FlowRunEntity flowRun = new FlowRunEntity();
+ flowRun.setUser(userId);
+ flowRun.setName(flowId);
+ flowRun.setRunId(flowRunId);
+
+ // read the start time
+ Long startTime = (Long)FlowRunColumn.MIN_START_TIME.readResult(result);
+ if (startTime != null) {
+ flowRun.setStartTime(startTime);
+ }
+ // read the end time if available
+ Long endTime = (Long)FlowRunColumn.MAX_END_TIME.readResult(result);
+ if (endTime != null) {
+ flowRun.setMaxEndTime(endTime);
+ }
+
+ // read the flow version
+ String version = (String)FlowRunColumn.FLOW_VERSION.readResult(result);
+ if (version != null) {
+ flowRun.setVersion(version);
+ }
+
+ // read metrics
+ readMetrics(flowRun, result, FlowRunColumnPrefix.METRIC);
+
+ // set the id
+ flowRun.setId(flowRun.getId());
+ return flowRun;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d8d8fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
new file mode 100644
index 0000000..466914b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
@@ -0,0 +1,389 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for generic entities that are stored in the entity
+ * table.
+ */
+class GenericEntityReader extends TimelineEntityReader {
+ private static final EntityTable ENTITY_TABLE = new EntityTable();
+ private static final Log LOG = LogFactory.getLog(GenericEntityReader.class);
+
+ private static final long DEFAULT_BEGIN_TIME = 0L;
+ private static final long DEFAULT_END_TIME = Long.MAX_VALUE;
+
+ /**
+ * Used to look up the flow context.
+ */
+ private final AppToFlowTable appToFlowTable = new AppToFlowTable();
+
+ public GenericEntityReader(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+ eventFilters, fieldsToRetrieve);
+ }
+
+ public GenericEntityReader(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowId, flowRunId, appId, entityType, entityId,
+ fieldsToRetrieve);
+ }
+
+ /**
+ * Uses the {@link EntityTable}.
+ */
+ protected BaseTable<?> getTable() {
+ return ENTITY_TABLE;
+ }
+
+ private FlowContext lookupFlowContext(String clusterId, String appId,
+ Configuration hbaseConf, Connection conn) throws IOException {
+ byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
+ Get get = new Get(rowKey);
+ Result result = appToFlowTable.getResult(hbaseConf, conn, get);
+ if (result != null && !result.isEmpty()) {
+ return new FlowContext(
+ AppToFlowColumn.FLOW_ID.readResult(result).toString(),
+ ((Number)AppToFlowColumn.FLOW_RUN_ID.readResult(result)).longValue());
+ } else {
+ throw new IOException(
+ "Unable to find the context flow ID and flow run ID for clusterId=" +
+ clusterId + ", appId=" + appId);
+ }
+ }
+
+ private static class FlowContext {
+ private final String flowId;
+ private final Long flowRunId;
+ public FlowContext(String flowId, Long flowRunId) {
+ this.flowId = flowId;
+ this.flowRunId = flowRunId;
+ }
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(userId, "userId shouldn't be null");
+ Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ Preconditions.checkNotNull(appId, "appId shouldn't be null");
+ Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
+ if (singleEntityRead) {
+ Preconditions.checkNotNull(entityId, "entityId shouldn't be null");
+ }
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ // In reality both should be null or neither should be null
+ if (flowId == null || flowRunId == null) {
+ FlowContext context =
+ lookupFlowContext(clusterId, appId, hbaseConf, conn);
+ flowId = context.flowId;
+ flowRunId = context.flowRunId;
+ }
+ if (fieldsToRetrieve == null) {
+ fieldsToRetrieve = EnumSet.noneOf(Field.class);
+ }
+ if (!singleEntityRead) {
+ if (limit == null || limit < 0) {
+ limit = TimelineReader.DEFAULT_LIMIT;
+ }
+ if (createdTimeBegin == null) {
+ createdTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (createdTimeEnd == null) {
+ createdTimeEnd = DEFAULT_END_TIME;
+ }
+ if (modifiedTimeBegin == null) {
+ modifiedTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (modifiedTimeEnd == null) {
+ modifiedTimeEnd = DEFAULT_END_TIME;
+ }
+ }
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ byte[] rowKey =
+ EntityRowKey.getRowKey(clusterId, userId, flowId, flowRunId, appId,
+ entityType, entityId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ return table.getResult(hbaseConf, conn, get);
+ }
+
+ @Override
+ protected Iterable<Result> getResults(Configuration hbaseConf,
+ Connection conn) throws IOException {
+ // Scan through part of the table to find the entities belong to one app
+ // and one type
+ Scan scan = new Scan();
+ scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
+ clusterId, userId, flowId, flowRunId, appId, entityType));
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ return table.getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ TimelineEntity entity = new TimelineEntity();
+ String entityType = EntityColumn.TYPE.readResult(result).toString();
+ entity.setType(entityType);
+ String entityId = EntityColumn.ID.readResult(result).toString();
+ entity.setId(entityId);
+
+ // fetch created time
+ Number createdTime = (Number)EntityColumn.CREATED_TIME.readResult(result);
+ entity.setCreatedTime(createdTime.longValue());
+ if (!singleEntityRead && (entity.getCreatedTime() < createdTimeBegin ||
+ entity.getCreatedTime() > createdTimeEnd)) {
+ return null;
+ }
+
+ // fetch modified time
+ Number modifiedTime = (Number)EntityColumn.MODIFIED_TIME.readResult(result);
+ entity.setModifiedTime(modifiedTime.longValue());
+ if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin ||
+ entity.getModifiedTime() > modifiedTimeEnd)) {
+ return null;
+ }
+
+ // fetch is related to entities
+ boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
+ readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO, true);
+ if (checkIsRelatedTo && !TimelineReaderUtils.matchRelations(
+ entity.getIsRelatedToEntities(), isRelatedTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
+ entity.getIsRelatedToEntities().clear();
+ }
+ }
+
+ // fetch relates to entities
+ boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
+ readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
+ if (checkRelatesTo && !TimelineReaderUtils.matchRelations(
+ entity.getRelatesToEntities(), relatesTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.RELATES_TO)) {
+ entity.getRelatesToEntities().clear();
+ }
+ }
+
+ // fetch info
+ boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
+ if (checkInfo &&
+ !TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.INFO)) {
+ entity.getInfo().clear();
+ }
+ }
+
+ // fetch configs
+ boolean checkConfigs = configFilters != null && configFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
+ if (checkConfigs && !TimelineReaderUtils.matchFilters(
+ entity.getConfigs(), configFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.CONFIGS)) {
+ entity.getConfigs().clear();
+ }
+ }
+
+ // fetch events
+ boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
+ readEvents(entity, result, false);
+ if (checkEvents && !TimelineReaderUtils.matchEventFilters(
+ entity.getEvents(), eventFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.EVENTS)) {
+ entity.getEvents().clear();
+ }
+ }
+
+ // fetch metrics
+ boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
+ readMetrics(entity, result, EntityColumnPrefix.METRIC);
+ if (checkMetrics && !TimelineReaderUtils.matchMetricFilters(
+ entity.getMetrics(), metricFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.METRICS)) {
+ entity.getMetrics().clear();
+ }
+ }
+ return entity;
+ }
+
+ /**
+ * Helper method for reading relationship.
+ */
+ protected <T> void readRelationship(
+ TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
+ boolean isRelatedTo) throws IOException {
+ // isRelatedTo and relatesTo are of type Map<String, Set<String>>
+ Map<String, Object> columns = prefix.readResults(result);
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ for (String id : Separator.VALUES.splitEncoded(
+ column.getValue().toString())) {
+ if (isRelatedTo) {
+ entity.addIsRelatedToEntity(column.getKey(), id);
+ } else {
+ entity.addRelatesToEntity(column.getKey(), id);
+ }
+ }
+ }
+ }
+
+ /**
+ * Helper method for reading key-value pairs for either info or config.
+ */
+ protected <T> void readKeyValuePairs(
+ TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
+ boolean isConfig) throws IOException {
+ // info and configuration are of type Map<String, Object or String>
+ Map<String, Object> columns = prefix.readResults(result);
+ if (isConfig) {
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ entity.addConfig(column.getKey(), column.getValue().toString());
+ }
+ } else {
+ entity.addInfo(columns);
+ }
+ }
+
+ /**
+ * Read events from the entity table or the application table. The column name
+ * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
+ * if there is no info associated with the event.
+ *
+ * See {@link EntityTable} and {@link ApplicationTable} for a more detailed
+ * schema description.
+ */
+ protected void readEvents(TimelineEntity entity, Result result,
+ boolean isApplication) throws IOException {
+ Map<String, TimelineEvent> eventsMap = new HashMap<>();
+ Map<?, Object> eventsResult = isApplication ?
+ ApplicationColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result) :
+ EntityColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
+ for (Map.Entry<?, Object> eventResult : eventsResult.entrySet()) {
+ byte[][] karr = (byte[][])eventResult.getKey();
+ // the column name is of the form "eventId=timestamp=infoKey"
+ if (karr.length == 3) {
+ String id = Bytes.toString(karr[0]);
+ long ts = TimelineWriterUtils.invert(Bytes.toLong(karr[1]));
+ String key = Separator.VALUES.joinEncoded(id, Long.toString(ts));
+ TimelineEvent event = eventsMap.get(key);
+ if (event == null) {
+ event = new TimelineEvent();
+ event.setId(id);
+ event.setTimestamp(ts);
+ eventsMap.put(key, event);
+ }
+ // handle empty info
+ String infoKey = karr[2].length == 0 ? null : Bytes.toString(karr[2]);
+ if (infoKey != null) {
+ event.addInfo(infoKey, eventResult.getValue());
+ }
+ } else {
+ LOG.warn("incorrectly formatted column name: it will be discarded");
+ continue;
+ }
+ }
+ Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
+ entity.addEvents(eventsSet);
+ }
+}
[32/50] [abbrv] hadoop git commit: YARN-3904. Refactor
timelineservice.storage to add support to online and offline aggregation
writers (Li Lu via sjlee)
Posted by sj...@apache.org.
YARN-3904. Refactor timelineservice.storage to add support to online and offline aggregation writers (Li Lu via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0fd93f15
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0fd93f15
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0fd93f15
Branch: refs/heads/YARN-2928-rebase
Commit: 0fd93f15781050aa1bad9834e7774d7b3a4c2983
Parents: aa927d8
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Aug 17 16:48:58 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:42 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 7 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +
.../storage/OfflineAggregationWriter.java | 66 +++
.../PhoenixOfflineAggregationWriterImpl.java | 356 +++++++++++++
.../storage/PhoenixTimelineWriterImpl.java | 530 -------------------
.../storage/TimelineSchemaCreator.java | 45 +-
.../storage/common/OfflineAggregationInfo.java | 110 ++++
...TestPhoenixOfflineAggregationWriterImpl.java | 162 ++++++
.../storage/TestPhoenixTimelineWriterImpl.java | 152 ------
.../storage/TestTimelineWriterImpl.java | 74 ---
11 files changed, 754 insertions(+), 761 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d4e2c4f..374b254 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -88,6 +88,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3906. Split the application table from the entity table. (Sangjin Lee
via junping_du)
+ YARN-3904. Refactor timelineservice.storage to add support to online and
+ offline aggregation writers (Li Lu via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 5583cd6..691170e 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -488,13 +488,12 @@
<!-- Ignore SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING warnings for Timeline Phoenix storage. -->
<!-- Since we're using dynamic columns, we have to generate SQL statements dynamically -->
<Match>
- <Class name="org.apache.hadoop.yarn.server.timelineservice.storage.PhoenixTimelineWriterImpl" />
+ <Class name="org.apache.hadoop.yarn.server.timelineservice.storage.PhoenixOfflineAggregationWriterImpl" />
<Or>
<Method name="storeEntityVariableLengthFields" />
- <Method name="storeEvents" />
- <Method name="storeMetrics" />
- <Method name="write" />
+ <Method name="writeAggregatedEntity" />
</Or>
+ <Bug pattern="SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING" />
</Match>
<!-- Following fields are used in ErrorsAndWarningsBlock, which is not a part of analysis of findbugs -->
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index e2e0635..073a8b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1783,6 +1783,16 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME =
7*24*60*60*1000; // 7 days
+ // Timeline service v2 offlien aggregation related keys
+ public static final String TIMELINE_OFFLINE_AGGREGATION_PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "aggregation.offline.";
+ public static final String PHOENIX_OFFLINE_STORAGE_CONN_STR
+ = TIMELINE_OFFLINE_AGGREGATION_PREFIX
+ + "phoenix.connectionString";
+
+ public static final String PHOENIX_OFFLINE_STORAGE_CONN_STR_DEFAULT
+ = "jdbc:phoenix:localhost:2181:/hbase";
+
// ///////////////////////////////
// Shared Cache Configs
// ///////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java
new file mode 100644
index 0000000..e1219e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.OfflineAggregationInfo;
+
+import java.io.IOException;
+
+/**
+ * YARN timeline service v2 offline aggregation storage interface
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public abstract class OfflineAggregationWriter extends AbstractService {
+
+ /**
+ * Construct the offline writer.
+ *
+ * @param name service name
+ */
+ public OfflineAggregationWriter(String name) {
+ super(name);
+ }
+
+ /**
+ * Persist aggregated timeline entities to the offline store based on which
+ * track this entity is to be rolled up to. The tracks along which aggregations
+ * are to be done are given by {@link OfflineAggregationInfo}.
+ *
+ * @param context a {@link TimelineCollectorContext} object that describes the
+ * context information of the aggregated data. Depends on the
+ * type of the aggregation, some fields of this context maybe
+ * empty or null.
+ * @param entities {@link TimelineEntities} to be persisted.
+ * @param info an {@link OfflineAggregationInfo} object that describes the
+ * detail of the aggregation. Current supported option is
+ * {@link OfflineAggregationInfo#FLOW_AGGREGATION}.
+ * @return a {@link TimelineWriteResponse} object.
+ * @throws IOException
+ */
+ abstract TimelineWriteResponse writeAggregatedEntity(
+ TimelineCollectorContext context,
+ TimelineEntities entities, OfflineAggregationInfo info) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
new file mode 100644
index 0000000..4c1352c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
@@ -0,0 +1,356 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.OfflineAggregationInfo;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.phoenix.util.PropertiesUtil;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * Offline aggregation Phoenix storage. This storage currently consists of two
+ * aggregation tables, one for flow level aggregation and one for user level
+ * aggregation.
+ *
+ * Example table record:
+ *
+ * <pre>
+ * |---------------------------|
+ * | Primary | Column Family|
+ * | key | metrics |
+ * |---------------------------|
+ * | row_key | metricId1: |
+ * | | metricValue1 |
+ * | | @timestamp1 |
+ * | | |
+ * | | metriciD1: |
+ * | | metricValue2 |
+ * | | @timestamp2 |
+ * | | |
+ * | | metricId2: |
+ * | | metricValue1 |
+ * | | @timestamp2 |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * |---------------------------|
+ * </pre>
+ *
+ * For the flow aggregation table, the primary key contains user, cluster, and
+ * flow id. For user aggregation table,the primary key is user.
+ *
+ * Metrics column family stores all aggregated metrics for each record.
+ */
+@Private
+@Unstable
+public class PhoenixOfflineAggregationWriterImpl
+ extends OfflineAggregationWriter {
+
+ private static final Log LOG
+ = LogFactory.getLog(PhoenixOfflineAggregationWriterImpl.class);
+ private static final String PHOENIX_COL_FAMILY_PLACE_HOLDER
+ = "timeline_cf_placeholder";
+
+ /** Default Phoenix JDBC driver name */
+ private static final String DRIVER_CLASS_NAME
+ = "org.apache.phoenix.jdbc.PhoenixDriver";
+
+ /** Default Phoenix timeline config column family */
+ private static final String METRIC_COLUMN_FAMILY = "m.";
+ /** Default Phoenix timeline info column family */
+ private static final String INFO_COLUMN_FAMILY = "i.";
+ /** Default separator for Phoenix storage */
+ private static final String AGGREGATION_STORAGE_SEPARATOR = ";";
+
+ /** Connection string to the deployed Phoenix cluster */
+ private String connString = null;
+ private Properties connProperties = new Properties();
+
+ public PhoenixOfflineAggregationWriterImpl(Properties prop) {
+ super(PhoenixOfflineAggregationWriterImpl.class.getName());
+ connProperties = PropertiesUtil.deepCopy(prop);
+ }
+
+ public PhoenixOfflineAggregationWriterImpl() {
+ super(PhoenixOfflineAggregationWriterImpl.class.getName());
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ Class.forName(DRIVER_CLASS_NAME);
+ // so check it here and only read in the config if it's not overridden.
+ connString =
+ conf.get(YarnConfiguration.PHOENIX_OFFLINE_STORAGE_CONN_STR,
+ YarnConfiguration.PHOENIX_OFFLINE_STORAGE_CONN_STR_DEFAULT);
+ super.init(conf);
+ }
+
+ @Override
+ public TimelineWriteResponse writeAggregatedEntity(
+ TimelineCollectorContext context, TimelineEntities entities,
+ OfflineAggregationInfo info) throws IOException {
+ TimelineWriteResponse response = new TimelineWriteResponse();
+ String sql = "UPSERT INTO " + info.getTableName()
+ + " (" + StringUtils.join(info.getPrimaryKeyList(), ",")
+ + ", created_time, modified_time, metric_names) "
+ + "VALUES ("
+ + StringUtils.repeat("?,", info.getPrimaryKeyList().length)
+ + "?, ?, ?)";
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("TimelineEntity write SQL: " + sql);
+ }
+
+ try (Connection conn = getConnection();
+ PreparedStatement ps = conn.prepareStatement(sql)) {
+ for (TimelineEntity entity : entities.getEntities()) {
+ HashMap<String, TimelineMetric> formattedMetrics = new HashMap<>();
+ if (entity.getMetrics() != null) {
+ for (TimelineMetric m : entity.getMetrics()) {
+ formattedMetrics.put(m.getId(), m);
+ }
+ }
+ int idx = info.setStringsForPrimaryKey(ps, context, null, 1);
+ ps.setLong(idx++, entity.getCreatedTime());
+ ps.setLong(idx++, entity.getModifiedTime());
+ ps.setString(idx++, StringUtils.join(formattedMetrics.keySet().toArray(),
+ AGGREGATION_STORAGE_SEPARATOR));
+ ps.execute();
+
+ storeEntityVariableLengthFields(entity, formattedMetrics, context, conn,
+ info);
+
+ conn.commit();
+ }
+ } catch (SQLException se) {
+ LOG.error("Failed to add entity to Phoenix " + se.getMessage());
+ throw new IOException(se);
+ } catch (Exception e) {
+ LOG.error("Exception on getting connection: " + e.getMessage());
+ throw new IOException(e);
+ }
+ return response;
+ }
+
+ /**
+ * Create Phoenix tables for offline aggregation storage if the tables do not
+ * exist.
+ *
+ * @throws IOException
+ */
+ public void createPhoenixTables() throws IOException {
+ // Create tables if necessary
+ try (Connection conn = getConnection();
+ Statement stmt = conn.createStatement()) {
+ // Table schema defined as in YARN-3817.
+ String sql = "CREATE TABLE IF NOT EXISTS "
+ + OfflineAggregationInfo.FLOW_AGGREGATION_TABLE_NAME
+ + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ + "flow_name VARCHAR NOT NULL, "
+ + "created_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
+ + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ + "metric_names VARCHAR, info_keys VARCHAR "
+ + "CONSTRAINT pk PRIMARY KEY("
+ + "user, cluster, flow_name))";
+ stmt.executeUpdate(sql);
+ sql = "CREATE TABLE IF NOT EXISTS "
+ + OfflineAggregationInfo.USER_AGGREGATION_TABLE_NAME
+ + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ + "created_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
+ + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ + "metric_names VARCHAR, info_keys VARCHAR "
+ + "CONSTRAINT pk PRIMARY KEY(user, cluster))";
+ stmt.executeUpdate(sql);
+ conn.commit();
+ } catch (SQLException se) {
+ LOG.error("Failed in init data " + se.getLocalizedMessage());
+ throw new IOException(se);
+ }
+ return;
+ }
+
+ // Utility functions
+ @Private
+ @VisibleForTesting
+ Connection getConnection() throws IOException {
+ Connection conn;
+ try {
+ conn = DriverManager.getConnection(connString, connProperties);
+ conn.setAutoCommit(false);
+ } catch (SQLException se) {
+ LOG.error("Failed to connect to phoenix server! "
+ + se.getLocalizedMessage());
+ throw new IOException(se);
+ }
+ return conn;
+ }
+
+ // WARNING: This method will permanently drop a table!
+ @Private
+ @VisibleForTesting
+ void dropTable(String tableName) throws Exception {
+ try (Connection conn = getConnection();
+ Statement stmt = conn.createStatement()) {
+ String sql = "DROP TABLE " + tableName;
+ stmt.executeUpdate(sql);
+ } catch (SQLException se) {
+ LOG.error("Failed in dropping entity table " + se.getLocalizedMessage());
+ throw se;
+ }
+ }
+
+ private static class DynamicColumns<K> {
+ static final String COLUMN_FAMILY_TYPE_BYTES = " VARBINARY";
+ static final String COLUMN_FAMILY_TYPE_STRING = " VARCHAR";
+ String columnFamilyPrefix;
+ String type;
+ Set<K> columns;
+
+ public DynamicColumns(String columnFamilyPrefix, String type,
+ Set<K> keyValues) {
+ this.columnFamilyPrefix = columnFamilyPrefix;
+ this.columns = keyValues;
+ this.type = type;
+ }
+ }
+
+ private static <K> StringBuilder appendColumnsSQL(
+ StringBuilder colNames, DynamicColumns<K> cfInfo) {
+ // Prepare the sql template by iterating through all keys
+ for (K key : cfInfo.columns) {
+ colNames.append(",").append(cfInfo.columnFamilyPrefix)
+ .append(key.toString()).append(cfInfo.type);
+ }
+ return colNames;
+ }
+
+ private static <K, V> int setValuesForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos,
+ boolean converToBytes) throws SQLException {
+ int idx = startPos;
+ for (Map.Entry<K, V> entry : keyValues.entrySet()) {
+ V value = entry.getValue();
+ if (value instanceof Collection) {
+ ps.setString(idx++, StringUtils.join(
+ (Collection) value, AGGREGATION_STORAGE_SEPARATOR));
+ } else {
+ if (converToBytes) {
+ try {
+ ps.setBytes(idx++, GenericObjectMapper.write(entry.getValue()));
+ } catch (IOException ie) {
+ LOG.error("Exception in converting values into bytes "
+ + ie.getMessage());
+ throw new SQLException(ie);
+ }
+ } else {
+ ps.setString(idx++, value.toString());
+ }
+ }
+ }
+ return idx;
+ }
+
+ private static <K, V> int setBytesForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos)
+ throws SQLException {
+ return setValuesForColumnFamily(ps, keyValues, startPos, true);
+ }
+
+ private static <K, V> int setStringsForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos)
+ throws SQLException {
+ return setValuesForColumnFamily(ps, keyValues, startPos, false);
+ }
+
+ private static void storeEntityVariableLengthFields(TimelineEntity entity,
+ Map<String, TimelineMetric> formattedMetrics,
+ TimelineCollectorContext context, Connection conn,
+ OfflineAggregationInfo aggregationInfo) throws SQLException {
+ int numPlaceholders = 0;
+ StringBuilder columnDefs = new StringBuilder(
+ StringUtils.join(aggregationInfo.getPrimaryKeyList(), ","));
+ if (formattedMetrics != null && formattedMetrics.size() > 0) {
+ appendColumnsSQL(columnDefs, new DynamicColumns<>(
+ METRIC_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_BYTES,
+ formattedMetrics.keySet()));
+ numPlaceholders += formattedMetrics.keySet().size();
+ }
+ if (numPlaceholders == 0) {
+ return;
+ }
+ StringBuilder placeholders = new StringBuilder();
+ placeholders.append(
+ StringUtils.repeat("?,", aggregationInfo.getPrimaryKeyList().length));
+ // numPlaceholders >= 1 now
+ placeholders.append("?")
+ .append(StringUtils.repeat(",?", numPlaceholders - 1));
+ String sqlVariableLengthFields = new StringBuilder("UPSERT INTO ")
+ .append(aggregationInfo.getTableName()).append(" (").append(columnDefs)
+ .append(") VALUES(").append(placeholders).append(")").toString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL statement for variable length fields: "
+ + sqlVariableLengthFields);
+ }
+ // Use try with resource statement for the prepared statement
+ try (PreparedStatement psVariableLengthFields =
+ conn.prepareStatement(sqlVariableLengthFields)) {
+ int idx = aggregationInfo.setStringsForPrimaryKey(
+ psVariableLengthFields, context, null, 1);
+ if (formattedMetrics != null && formattedMetrics.size() > 0) {
+ idx = setBytesForColumnFamily(
+ psVariableLengthFields, formattedMetrics, idx);
+ }
+ psVariableLengthFields.execute();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
deleted file mode 100644
index 381ff17..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
+++ /dev/null
@@ -1,530 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
-@Private
-@Unstable
-public class PhoenixTimelineWriterImpl extends AbstractService
- implements TimelineWriter {
-
- public static final String TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR
- = YarnConfiguration.TIMELINE_SERVICE_PREFIX
- + "writer.phoenix.connectionString";
-
- public static final String TIMELINE_SERVICE_PHEONIX_STORAGE_CONN_STR_DEFAULT
- = "jdbc:phoenix:localhost:2181:/hbase";
-
- private static final Log LOG
- = LogFactory.getLog(PhoenixTimelineWriterImpl.class);
- private static final String PHOENIX_COL_FAMILY_PLACE_HOLDER
- = "timeline_cf_placeholder";
- // These lists are not taking effects in table creations.
- private static final String[] PHOENIX_STORAGE_PK_LIST
- = {"cluster", "user", "flow_name", "flow_version", "flow_run", "app_id",
- "type", "entity_id"};
- private static final String[] TIMELINE_EVENT_EXTRA_PK_LIST =
- {"timestamp", "event_id"};
- private static final String[] TIMELINE_METRIC_EXTRA_PK_LIST =
- {"metric_id"};
- /** Default Phoenix JDBC driver name */
- private static final String DRIVER_CLASS_NAME
- = "org.apache.phoenix.jdbc.PhoenixDriver";
-
- /** Default Phoenix timeline entity table name */
- @VisibleForTesting
- static final String ENTITY_TABLE_NAME = "timeline_entity";
- /** Default Phoenix event table name */
- @VisibleForTesting
- static final String EVENT_TABLE_NAME = "timeline_event";
- /** Default Phoenix metric table name */
- @VisibleForTesting
- static final String METRIC_TABLE_NAME = "metric_singledata";
-
- /** Default Phoenix timeline config column family */
- private static final String CONFIG_COLUMN_FAMILY = "c.";
- /** Default Phoenix timeline info column family */
- private static final String INFO_COLUMN_FAMILY = "i.";
- /** Default Phoenix event info column family */
- private static final String EVENT_INFO_COLUMN_FAMILY = "ei.";
- /** Default Phoenix isRelatedTo column family */
- private static final String IS_RELATED_TO_FAMILY = "ir.";
- /** Default Phoenix relatesTo column family */
- private static final String RELATES_TO_FAMILY = "rt.";
- /** Default separator for Phoenix storage */
- private static final String PHOENIX_STORAGE_SEPARATOR = ";";
-
- /** Connection string to the deployed Phoenix cluster */
- @VisibleForTesting
- String connString = null;
- @VisibleForTesting
- Properties connProperties = new Properties();
-
- PhoenixTimelineWriterImpl() {
- super((PhoenixTimelineWriterImpl.class.getName()));
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- // so check it here and only read in the config if it's not overridden.
- connString =
- conf.get(TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR,
- TIMELINE_SERVICE_PHEONIX_STORAGE_CONN_STR_DEFAULT);
- createTables();
- super.init(conf);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- }
-
- @Override
- public TimelineWriteResponse write(String clusterId, String userId,
- String flowName, String flowVersion, long flowRunId, String appId,
- TimelineEntities entities) throws IOException {
- TimelineWriteResponse response = new TimelineWriteResponse();
- TimelineCollectorContext currContext = new TimelineCollectorContext(
- clusterId, userId, flowName, flowVersion, flowRunId, appId);
- String sql = "UPSERT INTO " + ENTITY_TABLE_NAME
- + " (" + StringUtils.join(PHOENIX_STORAGE_PK_LIST, ",")
- + ", creation_time, modified_time, configs) "
- + "VALUES (" + StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length)
- + "?, ?, ?)";
- if (LOG.isDebugEnabled()) {
- LOG.debug("TimelineEntity write SQL: " + sql);
- }
-
- try (Connection conn = getConnection();
- PreparedStatement ps = conn.prepareStatement(sql)) {
- for (TimelineEntity entity : entities.getEntities()) {
- int idx = setStringsForPrimaryKey(ps, currContext, entity, 1);
- ps.setLong(idx++, entity.getCreatedTime());
- ps.setLong(idx++, entity.getModifiedTime());
- String configKeys = StringUtils.join(
- entity.getConfigs().keySet(), PHOENIX_STORAGE_SEPARATOR);
- ps.setString(idx++, configKeys);
- ps.execute();
-
- storeEntityVariableLengthFields(entity, currContext, conn);
- storeEvents(entity, currContext, conn);
- storeMetrics(entity, currContext, conn);
-
- conn.commit();
- }
- } catch (SQLException se) {
- LOG.error("Failed to add entity to Phoenix " + se.getMessage());
- throw new IOException(se);
- } catch (Exception e) {
- LOG.error("Exception on getting connection: " + e.getMessage());
- throw new IOException(e);
- }
- return response;
- }
-
- /**
- * Aggregates the entity information to the timeline store based on which
- * track this entity is to be rolled up to The tracks along which aggregations
- * are to be done are given by {@link TimelineAggregationTrack}
- *
- * Any errors occurring for individual write request objects will be reported
- * in the response.
- *
- * @param data
- * a {@link TimelineEntity} object
- * a {@link TimelineAggregationTrack} enum value
- * @return a {@link TimelineWriteResponse} object.
- * @throws IOException
- */
- @Override
- public TimelineWriteResponse aggregate(TimelineEntity data,
- TimelineAggregationTrack track) throws IOException {
- return null;
-
- }
-
- @Override
- public void flush() throws IOException {
- // currently no-op
- }
-
- // Utility functions
- @Private
- @VisibleForTesting
- Connection getConnection() throws IOException {
- Connection conn;
- try {
- Class.forName(DRIVER_CLASS_NAME);
- conn = DriverManager.getConnection(connString, connProperties);
- conn.setAutoCommit(false);
- } catch (SQLException se) {
- LOG.error("Failed to connect to phoenix server! "
- + se.getLocalizedMessage());
- throw new IOException(se);
- } catch (ClassNotFoundException e) {
- LOG.error("Class not found! " + e.getLocalizedMessage());
- throw new IOException(e);
- }
- return conn;
- }
-
- private void createTables() throws Exception {
- // Create tables if necessary
- try (Connection conn = getConnection();
- Statement stmt = conn.createStatement()) {
- // Table schema defined as in YARN-3134.
- String sql = "CREATE TABLE IF NOT EXISTS " + ENTITY_TABLE_NAME
- + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
- + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
- + "flow_run UNSIGNED_LONG NOT NULL, "
- + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
- + "entity_id VARCHAR NOT NULL, "
- + "creation_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
- + "configs VARCHAR, "
- + CONFIG_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR, "
- + INFO_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
- + IS_RELATED_TO_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR, "
- + RELATES_TO_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR "
- + "CONSTRAINT pk PRIMARY KEY("
- + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
- + "type, entity_id))";
- stmt.executeUpdate(sql);
- sql = "CREATE TABLE IF NOT EXISTS " + EVENT_TABLE_NAME
- + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
- + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
- + "flow_run UNSIGNED_LONG NOT NULL, "
- + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
- + "entity_id VARCHAR NOT NULL, "
- + "timestamp UNSIGNED_LONG NOT NULL, event_id VARCHAR NOT NULL, "
- + EVENT_INFO_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY "
- + "CONSTRAINT pk PRIMARY KEY("
- + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
- + "type, entity_id, timestamp DESC, event_id))";
- stmt.executeUpdate(sql);
- sql = "CREATE TABLE IF NOT EXISTS " + METRIC_TABLE_NAME
- + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
- + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
- + "flow_run UNSIGNED_LONG NOT NULL, "
- + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
- + "entity_id VARCHAR NOT NULL, "
- + "metric_id VARCHAR NOT NULL, "
- + "singledata VARBINARY, "
- + "time UNSIGNED_LONG "
- + "CONSTRAINT pk PRIMARY KEY("
- + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
- + "type, entity_id, metric_id))";
- stmt.executeUpdate(sql);
- conn.commit();
- } catch (SQLException se) {
- LOG.error("Failed in init data " + se.getLocalizedMessage());
- throw se;
- }
- return;
- }
-
- private static class DynamicColumns<K> {
- static final String COLUMN_FAMILY_TYPE_BYTES = " VARBINARY";
- static final String COLUMN_FAMILY_TYPE_STRING = " VARCHAR";
- String columnFamilyPrefix;
- String type;
- Set<K> columns;
-
- public DynamicColumns(String columnFamilyPrefix, String type,
- Set<K> keyValues) {
- this.columnFamilyPrefix = columnFamilyPrefix;
- this.columns = keyValues;
- this.type = type;
- }
- }
-
- private static <K> StringBuilder appendColumnsSQL(
- StringBuilder colNames, DynamicColumns<K> cfInfo) {
- // Prepare the sql template by iterating through all keys
- for (K key : cfInfo.columns) {
- colNames.append(",").append(cfInfo.columnFamilyPrefix)
- .append(key.toString()).append(cfInfo.type);
- }
- return colNames;
- }
-
- private static <K, V> int setValuesForColumnFamily(
- PreparedStatement ps, Map<K, V> keyValues, int startPos,
- boolean converToBytes) throws SQLException {
- int idx = startPos;
- for (Map.Entry<K, V> entry : keyValues.entrySet()) {
- V value = entry.getValue();
- if (value instanceof Collection) {
- ps.setString(idx++, StringUtils.join(
- (Collection) value, PHOENIX_STORAGE_SEPARATOR));
- } else {
- if (converToBytes) {
- try {
- ps.setBytes(idx++, GenericObjectMapper.write(entry.getValue()));
- } catch (IOException ie) {
- LOG.error("Exception in converting values into bytes "
- + ie.getMessage());
- throw new SQLException(ie);
- }
- } else {
- ps.setString(idx++, value.toString());
- }
- }
- }
- return idx;
- }
-
- private static <K, V> int setBytesForColumnFamily(
- PreparedStatement ps, Map<K, V> keyValues, int startPos)
- throws SQLException {
- return setValuesForColumnFamily(ps, keyValues, startPos, true);
- }
-
- private static <K, V> int setStringsForColumnFamily(
- PreparedStatement ps, Map<K, V> keyValues, int startPos)
- throws SQLException {
- return setValuesForColumnFamily(ps, keyValues, startPos, false);
- }
-
- private static int setStringsForPrimaryKey(PreparedStatement ps,
- TimelineCollectorContext context, TimelineEntity entity, int startPos)
- throws SQLException {
- int idx = startPos;
- ps.setString(idx++, context.getClusterId());
- ps.setString(idx++, context.getUserId());
- ps.setString(idx++,
- context.getFlowName());
- ps.setString(idx++, context.getFlowVersion());
- ps.setLong(idx++, context.getFlowRunId());
- ps.setString(idx++, context.getAppId());
- ps.setString(idx++, entity.getType());
- ps.setString(idx++, entity.getId());
- return idx;
- }
-
- private static void storeEntityVariableLengthFields(TimelineEntity entity,
- TimelineCollectorContext context, Connection conn) throws SQLException {
- int numPlaceholders = 0;
- StringBuilder columnDefs = new StringBuilder(
- StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
- if (entity.getConfigs() != null) {
- Set<String> keySet = entity.getConfigs().keySet();
- appendColumnsSQL(columnDefs, new DynamicColumns<>(
- CONFIG_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
- keySet));
- numPlaceholders += keySet.size();
- }
- if (entity.getInfo() != null) {
- Set<String> keySet = entity.getInfo().keySet();
- appendColumnsSQL(columnDefs, new DynamicColumns<>(
- INFO_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_BYTES,
- keySet));
- numPlaceholders += keySet.size();
- }
- if (entity.getIsRelatedToEntities() != null) {
- Set<String> keySet = entity.getIsRelatedToEntities().keySet();
- appendColumnsSQL(columnDefs, new DynamicColumns<>(
- IS_RELATED_TO_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
- keySet));
- numPlaceholders += keySet.size();
- }
- if (entity.getRelatesToEntities() != null) {
- Set<String> keySet = entity.getRelatesToEntities().keySet();
- appendColumnsSQL(columnDefs, new DynamicColumns<>(
- RELATES_TO_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
- keySet));
- numPlaceholders += keySet.size();
- }
- if (numPlaceholders == 0) {
- return;
- }
- StringBuilder placeholders = new StringBuilder();
- placeholders.append(
- StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length));
- // numPlaceholders >= 1 now
- placeholders.append("?")
- .append(StringUtils.repeat(",?", numPlaceholders - 1));
- String sqlVariableLengthFields = new StringBuilder("UPSERT INTO ")
- .append(ENTITY_TABLE_NAME).append(" (").append(columnDefs)
- .append(") VALUES(").append(placeholders).append(")").toString();
- if (LOG.isDebugEnabled()) {
- LOG.debug("SQL statement for variable length fields: "
- + sqlVariableLengthFields);
- }
- // Use try with resource statement for the prepared statement
- try (PreparedStatement psVariableLengthFields =
- conn.prepareStatement(sqlVariableLengthFields)) {
- int idx = setStringsForPrimaryKey(
- psVariableLengthFields, context, entity, 1);
- if (entity.getConfigs() != null) {
- idx = setStringsForColumnFamily(
- psVariableLengthFields, entity.getConfigs(), idx);
- }
- if (entity.getInfo() != null) {
- idx = setBytesForColumnFamily(
- psVariableLengthFields, entity.getInfo(), idx);
- }
- if (entity.getIsRelatedToEntities() != null) {
- idx = setStringsForColumnFamily(
- psVariableLengthFields, entity.getIsRelatedToEntities(), idx);
- }
- if (entity.getRelatesToEntities() != null) {
- idx = setStringsForColumnFamily(
- psVariableLengthFields, entity.getRelatesToEntities(), idx);
- }
- psVariableLengthFields.execute();
- }
- }
-
- private static void storeMetrics(TimelineEntity entity,
- TimelineCollectorContext context, Connection conn) throws SQLException {
- if (entity.getMetrics() == null) {
- return;
- }
- Set<TimelineMetric> metrics = entity.getMetrics();
- for (TimelineMetric metric : metrics) {
- StringBuilder sqlColumns = new StringBuilder(
- StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
- sqlColumns.append(",")
- .append(StringUtils.join(TIMELINE_METRIC_EXTRA_PK_LIST, ","));
- sqlColumns.append(",").append("singledata, time");
- StringBuilder placeholders = new StringBuilder();
- placeholders.append(
- StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length))
- .append(StringUtils.repeat("?,", TIMELINE_METRIC_EXTRA_PK_LIST.length));
- placeholders.append("?, ?");
- String sqlMetric = new StringBuilder("UPSERT INTO ")
- .append(METRIC_TABLE_NAME).append(" (").append(sqlColumns)
- .append(") VALUES(").append(placeholders).append(")").toString();
- if (LOG.isDebugEnabled()) {
- LOG.debug("SQL statement for metric: " + sqlMetric);
- }
- try (PreparedStatement psMetrics = conn.prepareStatement(sqlMetric)) {
- if (metric.getType().equals(TimelineMetric.Type.TIME_SERIES)) {
- LOG.warn("The incoming timeline metric contains time series data, "
- + "which is currently not supported by Phoenix storage. "
- + "Time series will be truncated. ");
- }
- int idx = setStringsForPrimaryKey(psMetrics, context, entity, 1);
- psMetrics.setString(idx++, metric.getId());
- Iterator<Map.Entry<Long, Number>> currNumIter =
- metric.getValues().entrySet().iterator();
- if (currNumIter.hasNext()) {
- // TODO: support time series storage
- Map.Entry<Long, Number> currEntry = currNumIter.next();
- psMetrics.setBytes(idx++,
- GenericObjectMapper.write(currEntry.getValue()));
- psMetrics.setLong(idx++, currEntry.getKey());
- } else {
- psMetrics.setBytes(idx++, GenericObjectMapper.write(null));
- LOG.warn("The incoming metric contains an empty value set. ");
- }
- psMetrics.execute();
- } catch (IOException ie) {
- LOG.error("Exception on converting single data to bytes: "
- + ie.getMessage());
- throw new SQLException(ie);
- }
- }
- }
-
- private static void storeEvents(TimelineEntity entity,
- TimelineCollectorContext context, Connection conn) throws SQLException {
- if (entity.getEvents() == null) {
- return;
- }
- Set<TimelineEvent> events = entity.getEvents();
- for (TimelineEvent event : events) {
- // We need this number to check if the incoming event's info field is empty
- int numPlaceholders = 0;
- StringBuilder sqlColumns = new StringBuilder(
- StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
- sqlColumns.append(",")
- .append(StringUtils.join(TIMELINE_EVENT_EXTRA_PK_LIST, ","));
- appendColumnsSQL(sqlColumns, new DynamicColumns<>(
- EVENT_INFO_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_BYTES,
- event.getInfo().keySet()));
- numPlaceholders += event.getInfo().keySet().size();
- if (numPlaceholders == 0) {
- continue;
- }
- StringBuilder placeholders = new StringBuilder();
- placeholders.append(
- StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length))
- .append(StringUtils.repeat("?,", TIMELINE_EVENT_EXTRA_PK_LIST.length));
- // numPlaceholders >= 1 now
- placeholders.append("?")
- .append(StringUtils.repeat(",?", numPlaceholders - 1));
- String sqlEvents = new StringBuilder("UPSERT INTO ")
- .append(EVENT_TABLE_NAME).append(" (").append(sqlColumns)
- .append(") VALUES(").append(placeholders).append(")").toString();
- if (LOG.isDebugEnabled()) {
- LOG.debug("SQL statement for events: " + sqlEvents);
- }
- try (PreparedStatement psEvent = conn.prepareStatement(sqlEvents)) {
- int idx = setStringsForPrimaryKey(psEvent, context, entity, 1);
- psEvent.setLong(idx++, event.getTimestamp());
- psEvent.setString(idx++, event.getId());
- setBytesForColumnFamily(psEvent, event.getInfo(), idx);
- psEvent.execute();
- }
- }
- }
-
- // WARNING: This method will permanently drop a table!
- @Private
- @VisibleForTesting
- void dropTable(String tableName) throws Exception {
- try (Connection conn = getConnection();
- Statement stmt = conn.createStatement()) {
- String sql = "DROP TABLE " + tableName;
- stmt.executeUpdate(sql);
- } catch (SQLException se) {
- LOG.error("Failed in dropping entity table " + se.getLocalizedMessage());
- throw se;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index 3a22ed6..5120856 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -51,6 +53,7 @@ public class TimelineSchemaCreator {
final static String NAME = TimelineSchemaCreator.class.getSimpleName();
private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
+ private static final String PHOENIX_OPTION_SHORT = "p";
public static void main(String[] args) throws Exception {
@@ -83,7 +86,41 @@ public class TimelineSchemaCreator {
hbaseConf.set(ApplicationTable.TABLE_NAME_CONF_NAME,
applicationTableName);
}
- createAllTables(hbaseConf);
+
+ List<Exception> exceptions = new ArrayList<>();
+ try {
+ createAllTables(hbaseConf);
+ LOG.info("Successfully created HBase schema. ");
+ } catch (IOException e) {
+ LOG.error("Error in creating hbase tables: " + e.getMessage());
+ exceptions.add(e);
+ }
+
+ // Create Phoenix data schema if needed
+ if (commandLine.hasOption(PHOENIX_OPTION_SHORT)) {
+ Configuration phoenixConf = new Configuration();
+ try {
+ PhoenixOfflineAggregationWriterImpl phoenixWriter =
+ new PhoenixOfflineAggregationWriterImpl();
+ phoenixWriter.init(phoenixConf);
+ phoenixWriter.start();
+ phoenixWriter.createPhoenixTables();
+ phoenixWriter.stop();
+ LOG.info("Successfully created Phoenix offline aggregation schema. ");
+ } catch (IOException e) {
+ LOG.error("Error in creating phoenix tables: " + e.getMessage());
+ exceptions.add(e);
+ }
+ }
+ if (exceptions.size() > 0) {
+ LOG.warn("Schema creation finished with the following exceptions");
+ for (Exception e : exceptions) {
+ LOG.warn(e.getMessage());
+ }
+ System.exit(-1);
+ } else {
+ LOG.info("Schema creation finished successfully");
+ }
}
/**
@@ -115,6 +152,12 @@ public class TimelineSchemaCreator {
o.setRequired(false);
options.addOption(o);
+ o = new Option(PHOENIX_OPTION_SHORT, "usePhoenix", false,
+ "create Phoenix offline aggregation tables");
+ // No need to set arg name since we do not need an argument here
+ o.setRequired(false);
+ options.addOption(o);
+
CommandLineParser parser = new PosixParser();
CommandLine commandLine = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java
new file mode 100644
index 0000000..16c03a3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Class to carry the offline aggregation information for storage level
+ * implementations. There are currently two predefined aggregation info
+ * instances that represent flow and user level offline aggregations. Depend on
+ * its implementation, a storage class may use an OfflineAggregationInfo object
+ * to decide behaviors dynamically.
+ */
+public final class OfflineAggregationInfo {
+ /**
+ * Default flow level aggregation table name
+ */
+ @VisibleForTesting
+ public static final String FLOW_AGGREGATION_TABLE_NAME
+ = "yarn_timeline_flow_aggregation";
+ /**
+ * Default user level aggregation table name
+ */
+ public static final String USER_AGGREGATION_TABLE_NAME
+ = "yarn_timeline_user_aggregation";
+
+ // These lists are not taking effects in table creations.
+ private static final String[] FLOW_AGGREGATION_PK_LIST =
+ { "user", "cluster", "flow_name" };
+ private static final String[] USER_AGGREGATION_PK_LIST = { "user", "cluster"};
+
+ private final String tableName;
+ private final String[] primaryKeyList;
+ private final PrimaryKeyStringSetter primaryKeyStringSetter;
+
+ private OfflineAggregationInfo(String table, String[] pkList,
+ PrimaryKeyStringSetter formatter) {
+ tableName = table;
+ primaryKeyList = pkList;
+ primaryKeyStringSetter = formatter;
+ }
+
+ private interface PrimaryKeyStringSetter {
+ int setValues(PreparedStatement ps, TimelineCollectorContext context,
+ String[] extraInfo, int startPos) throws SQLException;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String[] getPrimaryKeyList() {
+ return primaryKeyList.clone();
+ }
+
+ public int setStringsForPrimaryKey(PreparedStatement ps,
+ TimelineCollectorContext context, String[] extraInfo, int startPos)
+ throws SQLException {
+ return primaryKeyStringSetter.setValues(ps, context, extraInfo, startPos);
+ }
+
+ public static final OfflineAggregationInfo FLOW_AGGREGATION =
+ new OfflineAggregationInfo(FLOW_AGGREGATION_TABLE_NAME,
+ FLOW_AGGREGATION_PK_LIST, new PrimaryKeyStringSetter() {
+ @Override
+ public int setValues(PreparedStatement ps,
+ TimelineCollectorContext context, String[] extraInfo, int startPos)
+ throws SQLException {
+ int idx = startPos;
+ ps.setString(idx++, context.getUserId());
+ ps.setString(idx++, context.getClusterId());
+ ps.setString(idx++, context.getFlowName());
+ return idx;
+ }
+ });
+
+ public static final OfflineAggregationInfo USER_AGGREGATION =
+ new OfflineAggregationInfo(USER_AGGREGATION_TABLE_NAME,
+ USER_AGGREGATION_PK_LIST, new PrimaryKeyStringSetter() {
+ @Override
+ public int setValues(PreparedStatement ps,
+ TimelineCollectorContext context, String[] extraInfo, int startPos)
+ throws SQLException {
+ int idx = startPos;
+ ps.setString(idx++, context.getUserId());
+ ps.setString(idx++, context.getClusterId());
+ return idx;
+ }
+ });
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
new file mode 100644
index 0000000..de66a17
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.OfflineAggregationInfo;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.ReadOnlyProps;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+
+public class TestPhoenixOfflineAggregationWriterImpl extends BaseTest {
+ private static PhoenixOfflineAggregationWriterImpl storage;
+ private static final int BATCH_SIZE = 3;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ storage = setupPhoenixClusterAndWriterForTest(conf);
+ }
+
+ @Test(timeout = 90000)
+ public void testFlowLevelAggregationStorage() throws Exception {
+ testAggregator(OfflineAggregationInfo.FLOW_AGGREGATION);
+ }
+
+ @Test(timeout = 90000)
+ public void testUserLevelAggregationStorage() throws Exception {
+ testAggregator(OfflineAggregationInfo.USER_AGGREGATION);
+ }
+
+ @AfterClass
+ public static void cleanup() throws Exception {
+ storage.dropTable(OfflineAggregationInfo.FLOW_AGGREGATION_TABLE_NAME);
+ storage.dropTable(OfflineAggregationInfo.USER_AGGREGATION_TABLE_NAME);
+ tearDownMiniCluster();
+ }
+
+ private static PhoenixOfflineAggregationWriterImpl
+ setupPhoenixClusterAndWriterForTest(YarnConfiguration conf)
+ throws Exception{
+ Map<String, String> props = new HashMap<>();
+ // Must update config before starting server
+ props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
+ Boolean.FALSE.toString());
+ props.put("java.security.krb5.realm", "");
+ props.put("java.security.krb5.kdc", "");
+ props.put(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER,
+ Boolean.FALSE.toString());
+ props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
+ props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
+ // Make a small batch size to test multiple calls to reserve sequences
+ props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB,
+ Long.toString(BATCH_SIZE));
+ // Must update config before starting server
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+
+ // Change connection settings for test
+ conf.set(
+ YarnConfiguration.PHOENIX_OFFLINE_STORAGE_CONN_STR,
+ getUrl());
+ PhoenixOfflineAggregationWriterImpl
+ myWriter = new PhoenixOfflineAggregationWriterImpl(TEST_PROPERTIES);
+ myWriter.init(conf);
+ myWriter.start();
+ myWriter.createPhoenixTables();
+ return myWriter;
+ }
+
+ private static TimelineEntity getTestAggregationTimelineEntity() {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "hello1";
+ String type = "testAggregationType";
+ entity.setId(id);
+ entity.setType(type);
+ entity.setCreatedTime(1425016501000L);
+ entity.setModifiedTime(1425016502000L);
+
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId("HDFS_BYTES_READ");
+ metric.addValue(1425016501100L, 8000);
+ entity.addMetric(metric);
+
+ return entity;
+ }
+
+ private void testAggregator(OfflineAggregationInfo aggregationInfo)
+ throws Exception {
+ // Set up a list of timeline entities and write them back to Phoenix
+ int numEntity = 1;
+ TimelineEntities te = new TimelineEntities();
+ te.addEntity(getTestAggregationTimelineEntity());
+ TimelineCollectorContext context = new TimelineCollectorContext("cluster_1",
+ "user1", "testFlow", null, 0, null);
+ storage.writeAggregatedEntity(context, te,
+ aggregationInfo);
+
+ // Verify if we're storing all entities
+ String[] primaryKeyList = aggregationInfo.getPrimaryKeyList();
+ String sql = "SELECT COUNT(" + primaryKeyList[primaryKeyList.length - 1]
+ +") FROM " + aggregationInfo.getTableName();
+ verifySQLWithCount(sql, numEntity, "Number of entities should be ");
+ // Check metric
+ sql = "SELECT COUNT(m.HDFS_BYTES_READ) FROM "
+ + aggregationInfo.getTableName() + "(m.HDFS_BYTES_READ VARBINARY) ";
+ verifySQLWithCount(sql, numEntity,
+ "Number of entities with info should be ");
+ }
+
+
+ private void verifySQLWithCount(String sql, int targetCount, String message)
+ throws Exception {
+ try (
+ Statement stmt =
+ storage.getConnection().createStatement();
+ ResultSet rs = stmt.executeQuery(sql)) {
+ assertTrue("Result set empty on statement " + sql, rs.next());
+ assertNotNull("Fail to execute query " + sql, rs);
+ assertEquals(message + " " + targetCount, targetCount, rs.getInt(1));
+ } catch (SQLException se) {
+ fail("SQL exception on query: " + sql
+ + " With exception message: " + se.getLocalizedMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
deleted file mode 100644
index dece83d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.hadoop.hbase.IntegrationTestingUtility;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
-import org.apache.phoenix.query.BaseTest;
-import org.apache.phoenix.query.QueryServices;
-import org.apache.phoenix.util.PropertiesUtil;
-import org.apache.phoenix.util.ReadOnlyProps;
-
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-
-public class TestPhoenixTimelineWriterImpl extends BaseTest {
- private static PhoenixTimelineWriterImpl writer;
- private static final int BATCH_SIZE = 3;
-
- @BeforeClass
- public static void setup() throws Exception {
- YarnConfiguration conf = new YarnConfiguration();
- writer = setupPhoenixClusterAndWriterForTest(conf);
- }
-
- @Test(timeout = 90000)
- public void testPhoenixWriterBasic() throws Exception {
- // Set up a list of timeline entities and write them back to Phoenix
- int numEntity = 12;
- TimelineEntities te =
- TestTimelineWriterImpl.getStandardTestTimelineEntities(numEntity);
- writer.write("cluster_1", "user1", "testFlow", "version1", 1l, "app_test_1", te);
- // Verify if we're storing all entities
- String sql = "SELECT COUNT(entity_id) FROM "
- + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME;
- verifySQLWithCount(sql, numEntity, "Number of entities should be ");
- // Check config (half of all entities)
- sql = "SELECT COUNT(c.config) FROM "
- + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME + "(c.config VARCHAR) ";
- verifySQLWithCount(sql, (numEntity / 2),
- "Number of entities with config should be ");
- // Check info (half of all entities)
- sql = "SELECT COUNT(i.info1) FROM "
- + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME + "(i.info1 VARBINARY) ";
- verifySQLWithCount(sql, (numEntity / 2),
- "Number of entities with info should be ");
- // Check config and info (a quarter of all entities)
- sql = "SELECT COUNT(entity_id) FROM "
- + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME
- + "(c.config VARCHAR, i.info1 VARBINARY) "
- + "WHERE c.config IS NOT NULL AND i.info1 IS NOT NULL";
- verifySQLWithCount(sql, (numEntity / 4),
- "Number of entities with both config and info should be ");
- // Check relatesToEntities and isRelatedToEntities
- sql = "SELECT COUNT(entity_id) FROM "
- + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME
- + "(rt.testType VARCHAR, ir.testType VARCHAR) "
- + "WHERE rt.testType IS NOT NULL AND ir.testType IS NOT NULL";
- verifySQLWithCount(sql, numEntity - 2,
- "Number of entities with both relatesTo and isRelatedTo should be ");
- // Check event
- sql = "SELECT COUNT(entity_id) FROM "
- + PhoenixTimelineWriterImpl.EVENT_TABLE_NAME;
- verifySQLWithCount(sql, (numEntity / 4), "Number of events should be ");
- // Check metrics
- sql = "SELECT COUNT(entity_id) FROM "
- + PhoenixTimelineWriterImpl.METRIC_TABLE_NAME;
- verifySQLWithCount(sql, (numEntity / 4), "Number of events should be ");
- }
-
- @AfterClass
- public static void cleanup() throws Exception {
- writer.dropTable(PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME);
- writer.dropTable(PhoenixTimelineWriterImpl.EVENT_TABLE_NAME);
- writer.dropTable(PhoenixTimelineWriterImpl.METRIC_TABLE_NAME);
- writer.serviceStop();
- tearDownMiniCluster();
- }
-
- private static PhoenixTimelineWriterImpl setupPhoenixClusterAndWriterForTest(
- YarnConfiguration conf) throws Exception{
- Map<String, String> props = new HashMap<>();
- // Must update config before starting server
- props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
- Boolean.FALSE.toString());
- props.put("java.security.krb5.realm", "");
- props.put("java.security.krb5.kdc", "");
- props.put(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER,
- Boolean.FALSE.toString());
- props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
- props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
- // Make a small batch size to test multiple calls to reserve sequences
- props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB,
- Long.toString(BATCH_SIZE));
- // Must update config before starting server
- setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
-
- PhoenixTimelineWriterImpl myWriter = new PhoenixTimelineWriterImpl();
- // Change connection settings for test
- conf.set(
- PhoenixTimelineWriterImpl.TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR,
- getUrl());
- myWriter.connProperties = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- myWriter.serviceInit(conf);
- return myWriter;
- }
-
- private void verifySQLWithCount(String sql, int targetCount, String message)
- throws Exception {
- try (
- Statement stmt =
- writer.getConnection().createStatement();
- ResultSet rs = stmt.executeQuery(sql)) {
- assertTrue("Result set empty on statement " + sql, rs.next());
- assertNotNull("Fail to execute query " + sql, rs);
- assertEquals(message + " " + targetCount, targetCount, rs.getInt(1));
- } catch (SQLException se) {
- fail("SQL exception on query: " + sql
- + " With exception message: " + se.getLocalizedMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd93f15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
deleted file mode 100644
index 7a7afc0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-
-public class TestTimelineWriterImpl {
- static TimelineEntities getStandardTestTimelineEntities(int listSize) {
- TimelineEntities te = new TimelineEntities();
- for (int i = 0; i < listSize; i++) {
- TimelineEntity entity = new TimelineEntity();
- String id = "hello" + i;
- String type = "testType";
- entity.setId(id);
- entity.setType(type);
- entity.setCreatedTime(1425016501000L + i);
- entity.setModifiedTime(1425016502000L + i);
- if (i > 0) {
- entity.addRelatesToEntity(type, "hello" + i);
- entity.addRelatesToEntity(type, "hello" + (i - 1));
- }
- if (i < listSize - 1) {
- entity.addIsRelatedToEntity(type, "hello" + i);
- entity.addIsRelatedToEntity(type, "hello" + (i + 1));
- }
- int category = i % 4;
- switch (category) {
- case 0:
- entity.addConfig("config", "config" + i);
- // Fall through deliberately
- case 1:
- entity.addInfo("info1", new Integer(i));
- entity.addInfo("info2", "helloworld");
- // Fall through deliberately
- case 2:
- break;
- case 3:
- entity.addConfig("config", "config" + i);
- TimelineEvent event = new TimelineEvent();
- event.setId("test event");
- event.setTimestamp(1425016501100L + i);
- event.addInfo("test_info", "content for " + entity.getId());
- event.addInfo("test_info1", new Integer(i));
- entity.addEvent(event);
- TimelineMetric metric = new TimelineMetric();
- metric.setId("HDFS_BYTES_READ");
- metric.addValue(1425016501100L + i, 8000 + i);
- entity.addMetric(metric);
- break;
- }
- te.addEntity(entity);
- }
- return te;
- }
-}
[17/50] [abbrv] hadoop git commit: YARN-3801. [JDK-8] Exclude
jdk.tools from hbase-client and hbase-testing-util (Tsuyoshi Ozawa via sjlee)
Posted by sj...@apache.org.
YARN-3801. [JDK-8] Exclude jdk.tools from hbase-client and hbase-testing-util (Tsuyoshi Ozawa via sjlee)
(cherry picked from commit a1bb9137af84a34bde799f45e7ab8a21e33d55e0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50eaf2b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50eaf2b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50eaf2b4
Branch: refs/heads/YARN-2928-rebase
Commit: 50eaf2b41a6dc2e01facc45dddffcd4824637409
Parents: d35d861
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Jun 15 21:15:33 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:36 2015 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 10 ++++++++++
hadoop-yarn-project/CHANGES.txt | 3 +++
2 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50eaf2b4/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index e42b2c9..2771c3c 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1005,6 +1005,12 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>jdk.tools</artifactId>
+ <groupId>jdk.tools</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.phoenix</groupId>
@@ -1051,6 +1057,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>jdk.tools</artifactId>
+ <groupId>jdk.tools</groupId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50eaf2b4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 22860ea..a9e6d85 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -85,6 +85,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3044. Made RM write app, attempt and optional container lifecycle
events to timeline service v2. (Naganarasimha G R via zjshen)
+ YARN-3801. [JDK-8] Exclude jdk.tools from hbase-client and
+ hbase-testing-util (Tsuyoshi Ozawa via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
[26/50] [abbrv] hadoop git commit: MAPREDUCE-6337. Added a mode to
replay MR job history files and put them into the timeline service v2.
Contributed by Sangjin Lee.
Posted by sj...@apache.org.
MAPREDUCE-6337. Added a mode to replay MR job history files and put them into the timeline service v2. Contributed by Sangjin Lee.
(cherry picked from commit 463e070a8e7c882706a96eaa20ea49bfe9982875)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8349b90c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8349b90c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8349b90c
Branch: refs/heads/YARN-2928-rebase
Commit: 8349b90c211c32c767afed5c88127d426a3e9548
Parents: 491b302
Author: Zhijie Shen <zj...@apache.org>
Authored: Thu May 14 15:16:33 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:41 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../hadoop/mapred/JobHistoryFileParser.java | 53 ++++
.../mapred/JobHistoryFileReplayMapper.java | 301 +++++++++++++++++++
.../hadoop/mapred/SimpleEntityWriter.java | 139 +++++++++
.../hadoop/mapred/TimelineEntityConverter.java | 207 +++++++++++++
.../mapred/TimelineServicePerformanceV2.java | 191 ++++--------
.../collector/TimelineCollectorManager.java | 8 +-
.../storage/FileSystemTimelineWriterImpl.java | 23 +-
.../timelineservice/storage/package-info.java | 24 ++
9 files changed, 809 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8349b90c/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index b65876e..17d1c9f 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -12,6 +12,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
MAPREDUCE-6335. Created MR job based performance test driver for the
timeline service v2. (Sangjin Lee via zjshen)
+ MAPREDUCE-6337. Added a mode to replay MR job history files and put them
+ into the timeline service v2. (Sangjin Lee via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8349b90c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
new file mode 100644
index 0000000..9d051df
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+
+class JobHistoryFileParser {
+ private static final Log LOG = LogFactory.getLog(JobHistoryFileParser.class);
+
+ private final FileSystem fs;
+
+ public JobHistoryFileParser(FileSystem fs) {
+ LOG.info("JobHistoryFileParser created with " + fs);
+ this.fs = fs;
+ }
+
+ public JobInfo parseHistoryFile(Path path) throws IOException {
+ LOG.info("parsing job history file " + path);
+ JobHistoryParser parser = new JobHistoryParser(fs, path);
+ return parser.parse();
+ }
+
+ public Configuration parseConfiguration(Path path) throws IOException {
+ LOG.info("parsing job configuration file " + path);
+ Configuration conf = new Configuration(false);
+ conf.addResource(fs.open(path));
+ return conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8349b90c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
new file mode 100644
index 0000000..802b78f
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.mapred.TimelineServicePerformanceV2.EntityWriter;
+import org.apache.hadoop.mapred.TimelineServicePerformanceV2.PerfCounters;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+
+/**
+ * Mapper for TimelineServicePerformanceV2 that replays job history files to the
+ * timeline service.
+ *
+ */
+class JobHistoryFileReplayMapper extends EntityWriter {
+ private static final Log LOG =
+ LogFactory.getLog(JobHistoryFileReplayMapper.class);
+
+ static final String PROCESSING_PATH = "processing path";
+ static final String REPLAY_MODE = "replay mode";
+ static final int WRITE_ALL_AT_ONCE = 1;
+ static final int WRITE_PER_ENTITY = 2;
+ static final int REPLAY_MODE_DEFAULT = WRITE_ALL_AT_ONCE;
+
+ private static final Pattern JOB_ID_PARSER =
+ Pattern.compile("^(job_[0-9]+_([0-9]+)).*");
+
+ public static class JobFiles {
+ private final String jobId;
+ private Path jobHistoryFilePath;
+ private Path jobConfFilePath;
+
+ public JobFiles(String jobId) {
+ this.jobId = jobId;
+ }
+
+ public String getJobId() {
+ return jobId;
+ }
+
+ public Path getJobHistoryFilePath() {
+ return jobHistoryFilePath;
+ }
+
+ public void setJobHistoryFilePath(Path jobHistoryFilePath) {
+ this.jobHistoryFilePath = jobHistoryFilePath;
+ }
+
+ public Path getJobConfFilePath() {
+ return jobConfFilePath;
+ }
+
+ public void setJobConfFilePath(Path jobConfFilePath) {
+ this.jobConfFilePath = jobConfFilePath;
+ }
+
+ @Override
+ public int hashCode() {
+ return jobId.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ JobFiles other = (JobFiles) obj;
+ return jobId.equals(other.jobId);
+ }
+ }
+
+ private enum FileType { JOB_HISTORY_FILE, JOB_CONF_FILE, UNKNOWN }
+
+
+ @Override
+ protected void writeEntities(Configuration tlConf,
+ TimelineCollectorManager manager, Context context) throws IOException {
+ // collect the apps it needs to process
+ Configuration conf = context.getConfiguration();
+ int taskId = context.getTaskAttemptID().getTaskID().getId();
+ int size = conf.getInt(MRJobConfig.NUM_MAPS,
+ TimelineServicePerformanceV2.NUM_MAPS_DEFAULT);
+ String processingDir =
+ conf.get(JobHistoryFileReplayMapper.PROCESSING_PATH);
+ int replayMode =
+ conf.getInt(JobHistoryFileReplayMapper.REPLAY_MODE,
+ JobHistoryFileReplayMapper.REPLAY_MODE_DEFAULT);
+ Path processingPath = new Path(processingDir);
+ FileSystem processingFs = processingPath.getFileSystem(conf);
+ JobHistoryFileParser parser = new JobHistoryFileParser(processingFs);
+ TimelineEntityConverter converter = new TimelineEntityConverter();
+
+ Collection<JobFiles> jobs =
+ selectJobFiles(processingFs, processingPath, taskId, size);
+ if (jobs.isEmpty()) {
+ LOG.info(context.getTaskAttemptID().getTaskID() +
+ " will process no jobs");
+ } else {
+ LOG.info(context.getTaskAttemptID().getTaskID() + " will process " +
+ jobs.size() + " jobs");
+ }
+ for (JobFiles job: jobs) {
+ // process each job
+ String jobIdStr = job.getJobId();
+ LOG.info("processing " + jobIdStr + "...");
+ JobId jobId = TypeConverter.toYarn(JobID.forName(jobIdStr));
+ ApplicationId appId = jobId.getAppId();
+
+ // create the app level timeline collector and start it
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId);
+ manager.putIfAbsent(appId, collector);
+ try {
+ // parse the job info and configuration
+ JobInfo jobInfo =
+ parser.parseHistoryFile(job.getJobHistoryFilePath());
+ Configuration jobConf =
+ parser.parseConfiguration(job.getJobConfFilePath());
+ LOG.info("parsed the job history file and the configuration file for job"
+ + jobIdStr);
+
+ // set the context
+ // flow id: job name, flow run id: timestamp, user id
+ TimelineCollectorContext tlContext =
+ collector.getTimelineEntityContext();
+ tlContext.setFlowName(jobInfo.getJobname());
+ tlContext.setFlowRunId(jobInfo.getSubmitTime());
+ tlContext.setUserId(jobInfo.getUsername());
+
+ // create entities from job history and write them
+ long totalTime = 0;
+ Set<TimelineEntity> entitySet =
+ converter.createTimelineEntities(jobInfo, jobConf);
+ LOG.info("converted them into timeline entities for job " + jobIdStr);
+ // use the current user for this purpose
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ long startWrite = System.nanoTime();
+ try {
+ switch (replayMode) {
+ case JobHistoryFileReplayMapper.WRITE_ALL_AT_ONCE:
+ writeAllEntities(collector, entitySet, ugi);
+ break;
+ case JobHistoryFileReplayMapper.WRITE_PER_ENTITY:
+ writePerEntity(collector, entitySet, ugi);
+ break;
+ default:
+ break;
+ }
+ } catch (Exception e) {
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
+ increment(1);
+ LOG.error("writing to the timeline service failed", e);
+ }
+ long endWrite = System.nanoTime();
+ totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite-startWrite);
+ int numEntities = entitySet.size();
+ LOG.info("wrote " + numEntities + " entities in " + totalTime + " ms");
+
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
+ increment(totalTime);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
+ increment(numEntities);
+ } finally {
+ manager.remove(appId);
+ context.progress(); // move it along
+ }
+ }
+ }
+
+ private void writeAllEntities(AppLevelTimelineCollector collector,
+ Set<TimelineEntity> entitySet, UserGroupInformation ugi)
+ throws IOException {
+ TimelineEntities entities = new TimelineEntities();
+ entities.setEntities(entitySet);
+ collector.putEntities(entities, ugi);
+ }
+
+ private void writePerEntity(AppLevelTimelineCollector collector,
+ Set<TimelineEntity> entitySet, UserGroupInformation ugi)
+ throws IOException {
+ for (TimelineEntity entity : entitySet) {
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ collector.putEntities(entities, ugi);
+ LOG.info("wrote entity " + entity.getId());
+ }
+ }
+
+ private Collection<JobFiles> selectJobFiles(FileSystem fs,
+ Path processingRoot, int i, int size) throws IOException {
+ Map<String,JobFiles> jobs = new HashMap<>();
+ RemoteIterator<LocatedFileStatus> it = fs.listFiles(processingRoot, true);
+ while (it.hasNext()) {
+ LocatedFileStatus status = it.next();
+ Path path = status.getPath();
+ String fileName = path.getName();
+ Matcher m = JOB_ID_PARSER.matcher(fileName);
+ if (!m.matches()) {
+ continue;
+ }
+ String jobId = m.group(1);
+ int lastId = Integer.parseInt(m.group(2));
+ int mod = lastId % size;
+ if (mod != i) {
+ continue;
+ }
+ LOG.info("this mapper will process file " + fileName);
+ // it's mine
+ JobFiles jobFiles = jobs.get(jobId);
+ if (jobFiles == null) {
+ jobFiles = new JobFiles(jobId);
+ jobs.put(jobId, jobFiles);
+ }
+ setFilePath(fileName, path, jobFiles);
+ }
+ return jobs.values();
+ }
+
+ private void setFilePath(String fileName, Path path,
+ JobFiles jobFiles) {
+ // determine if we're dealing with a job history file or a job conf file
+ FileType type = getFileType(fileName);
+ switch (type) {
+ case JOB_HISTORY_FILE:
+ if (jobFiles.getJobHistoryFilePath() == null) {
+ jobFiles.setJobHistoryFilePath(path);
+ } else {
+ LOG.warn("we already have the job history file " +
+ jobFiles.getJobHistoryFilePath() + ": skipping " + path);
+ }
+ break;
+ case JOB_CONF_FILE:
+ if (jobFiles.getJobConfFilePath() == null) {
+ jobFiles.setJobConfFilePath(path);
+ } else {
+ LOG.warn("we already have the job conf file " +
+ jobFiles.getJobConfFilePath() + ": skipping " + path);
+ }
+ break;
+ case UNKNOWN:
+ LOG.warn("unknown type: " + path);
+ }
+ }
+
+ private FileType getFileType(String fileName) {
+ if (fileName.endsWith(".jhist")) {
+ return FileType.JOB_HISTORY_FILE;
+ }
+ if (fileName.endsWith("_conf.xml")) {
+ return FileType.JOB_CONF_FILE;
+ }
+ return FileType.UNKNOWN;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8349b90c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
new file mode 100644
index 0000000..4ef0a14
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.TimelineServicePerformanceV2.EntityWriter;
+import org.apache.hadoop.mapred.TimelineServicePerformanceV2.PerfCounters;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+
+/**
+ * Adds simple entities with random string payload, events, metrics, and
+ * configuration.
+ */
+class SimpleEntityWriter extends EntityWriter {
+ private static final Log LOG = LogFactory.getLog(SimpleEntityWriter.class);
+
+ // constants for mtype = 1
+ static final String KBS_SENT = "kbs sent";
+ static final int KBS_SENT_DEFAULT = 1;
+ static final String TEST_TIMES = "testtimes";
+ static final int TEST_TIMES_DEFAULT = 100;
+ static final String TIMELINE_SERVICE_PERFORMANCE_RUN_ID =
+ "timeline.server.performance.run.id";
+
+ protected void writeEntities(Configuration tlConf,
+ TimelineCollectorManager manager, Context context) throws IOException {
+ Configuration conf = context.getConfiguration();
+ // simulate the app id with the task id
+ int taskId = context.getTaskAttemptID().getTaskID().getId();
+ long timestamp = conf.getLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID, 0);
+ ApplicationId appId = ApplicationId.newInstance(timestamp, taskId);
+
+ // create the app level timeline collector
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId);
+ manager.putIfAbsent(appId, collector);
+
+ try {
+ // set the context
+ // flow id: job name, flow run id: timestamp, user id
+ TimelineCollectorContext tlContext =
+ collector.getTimelineEntityContext();
+ tlContext.setFlowName(context.getJobName());
+ tlContext.setFlowRunId(timestamp);
+ tlContext.setUserId(context.getUser());
+
+ final int kbs = conf.getInt(KBS_SENT, KBS_SENT_DEFAULT);
+
+ long totalTime = 0;
+ final int testtimes = conf.getInt(TEST_TIMES, TEST_TIMES_DEFAULT);
+ final Random rand = new Random();
+ final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
+ final char[] payLoad = new char[kbs * 1024];
+
+ for (int i = 0; i < testtimes; i++) {
+ // Generate a fixed length random payload
+ for (int xx = 0; xx < kbs * 1024; xx++) {
+ int alphaNumIdx =
+ rand.nextInt(TimelineServicePerformanceV2.alphaNums.length);
+ payLoad[xx] = TimelineServicePerformanceV2.alphaNums[alphaNumIdx];
+ }
+ String entId = taskAttemptId + "_" + Integer.toString(i);
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId(entId);
+ entity.setType("FOO_ATTEMPT");
+ entity.addInfo("PERF_TEST", payLoad);
+ // add an event
+ TimelineEvent event = new TimelineEvent();
+ event.setTimestamp(System.currentTimeMillis());
+ event.addInfo("foo_event", "test");
+ entity.addEvent(event);
+ // add a metric
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId("foo_metric");
+ metric.addValue(System.currentTimeMillis(), 123456789L);
+ entity.addMetric(metric);
+ // add a config
+ entity.addConfig("foo", "bar");
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ // use the current user for this purpose
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ long startWrite = System.nanoTime();
+ try {
+ collector.putEntities(entities, ugi);
+ } catch (Exception e) {
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
+ increment(1);
+ LOG.error("writing to the timeline service failed", e);
+ }
+ long endWrite = System.nanoTime();
+ totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite-startWrite);
+ }
+ LOG.info("wrote " + testtimes + " entities (" + kbs*testtimes +
+ " kB) in " + totalTime + " ms");
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
+ increment(totalTime);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
+ increment(testtimes);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).
+ increment(kbs*testtimes);
+ } finally {
+ // clean up
+ manager.remove(appId);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8349b90c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
new file mode 100644
index 0000000..80928dc
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+
+class TimelineEntityConverter {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineEntityConverter.class);
+
+ static final String JOB = "MAPREDUCE_JOB";
+ static final String TASK = "MAPREDUCE_TASK";
+ static final String TASK_ATTEMPT = "MAPREDUCE_TASK_ATTEMPT";
+
+ /**
+ * Creates job, task, and task attempt entities based on the job history info
+ * and configuration.
+ *
+ * Note: currently these are plan timeline entities created for mapreduce
+ * types. These are not meant to be the complete and accurate entity set-up
+ * for mapreduce jobs. We do not leverage hierarchical timeline entities. If
+ * we create canonical mapreduce hierarchical timeline entities with proper
+ * parent-child relationship, we could modify this to use that instead.
+ *
+ * Note that we also do not add info to the YARN application entity, which
+ * would be needed for aggregation.
+ */
+ public Set<TimelineEntity> createTimelineEntities(JobInfo jobInfo,
+ Configuration conf) {
+ Set<TimelineEntity> entities = new HashSet<>();
+
+ // create the job entity
+ TimelineEntity job = createJobEntity(jobInfo, conf);
+ entities.add(job);
+
+ // create the task and task attempt entities
+ Set<TimelineEntity> tasksAndAttempts =
+ createTaskAndTaskAttemptEntities(jobInfo);
+ entities.addAll(tasksAndAttempts);
+
+ return entities;
+ }
+
+ private TimelineEntity createJobEntity(JobInfo jobInfo, Configuration conf) {
+ TimelineEntity job = new TimelineEntity();
+ job.setType(JOB);
+ job.setId(jobInfo.getJobId().toString());
+ job.setCreatedTime(jobInfo.getSubmitTime());
+
+ job.addInfo("JOBNAME", jobInfo.getJobname());
+ job.addInfo("USERNAME", jobInfo.getUsername());
+ job.addInfo("JOB_QUEUE_NAME", jobInfo.getJobQueueName());
+ job.addInfo("SUBMIT_TIME", jobInfo.getSubmitTime());
+ job.addInfo("LAUNCH_TIME", jobInfo.getLaunchTime());
+ job.addInfo("FINISH_TIME", jobInfo.getFinishTime());
+ job.addInfo("JOB_STATUS", jobInfo.getJobStatus());
+ job.addInfo("PRIORITY", jobInfo.getPriority());
+ job.addInfo("TOTAL_MAPS", jobInfo.getTotalMaps());
+ job.addInfo("TOTAL_REDUCES", jobInfo.getTotalReduces());
+ job.addInfo("UBERIZED", jobInfo.getUberized());
+ job.addInfo("ERROR_INFO", jobInfo.getErrorInfo());
+
+ // add metrics from total counters
+ // we omit the map counters and reduce counters for now as it's kind of
+ // awkward to put them (map/reduce/total counters are really a group of
+ // related counters)
+ Counters totalCounters = jobInfo.getTotalCounters();
+ if (totalCounters != null) {
+ addMetrics(job, totalCounters);
+ }
+ // finally add configuration to the job
+ addConfiguration(job, conf);
+ LOG.info("converted job " + jobInfo.getJobId() + " to a timeline entity");
+ return job;
+ }
+
+ private void addConfiguration(TimelineEntity job, Configuration conf) {
+ for (Map.Entry<String,String> e: conf) {
+ job.addConfig(e.getKey(), e.getValue());
+ }
+ }
+
+ private void addMetrics(TimelineEntity entity, Counters counters) {
+ for (CounterGroup g: counters) {
+ String groupName = g.getName();
+ for (Counter c: g) {
+ String name = groupName + ":" + c.getName();
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId(name);
+ metric.addValue(System.currentTimeMillis(), c.getValue());
+ entity.addMetric(metric);
+ }
+ }
+ }
+
+ private Set<TimelineEntity> createTaskAndTaskAttemptEntities(JobInfo jobInfo) {
+ Set<TimelineEntity> entities = new HashSet<>();
+ Map<TaskID,TaskInfo> taskInfoMap = jobInfo.getAllTasks();
+ LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
+ " tasks");
+ for (TaskInfo taskInfo: taskInfoMap.values()) {
+ TimelineEntity task = createTaskEntity(taskInfo);
+ entities.add(task);
+ // add the task attempts from this task
+ Set<TimelineEntity> taskAttempts = createTaskAttemptEntities(taskInfo);
+ entities.addAll(taskAttempts);
+ }
+ return entities;
+ }
+
+ private TimelineEntity createTaskEntity(TaskInfo taskInfo) {
+ TimelineEntity task = new TimelineEntity();
+ task.setType(TASK);
+ task.setId(taskInfo.getTaskId().toString());
+ task.setCreatedTime(taskInfo.getStartTime());
+
+ task.addInfo("START_TIME", taskInfo.getStartTime());
+ task.addInfo("FINISH_TIME", taskInfo.getFinishTime());
+ task.addInfo("TASK_TYPE", taskInfo.getTaskType());
+ task.addInfo("TASK_STATUS", taskInfo.getTaskStatus());
+ task.addInfo("ERROR_INFO", taskInfo.getError());
+
+ // add metrics from counters
+ Counters counters = taskInfo.getCounters();
+ if (counters != null) {
+ addMetrics(task, counters);
+ }
+ LOG.info("converted task " + taskInfo.getTaskId() +
+ " to a timeline entity");
+ return task;
+ }
+
+ private Set<TimelineEntity> createTaskAttemptEntities(TaskInfo taskInfo) {
+ Set<TimelineEntity> taskAttempts = new HashSet<TimelineEntity>();
+ Map<TaskAttemptID,TaskAttemptInfo> taskAttemptInfoMap =
+ taskInfo.getAllTaskAttempts();
+ LOG.info("task " + taskInfo.getTaskId() + " has " +
+ taskAttemptInfoMap.size() + " task attempts");
+ for (TaskAttemptInfo taskAttemptInfo: taskAttemptInfoMap.values()) {
+ TimelineEntity taskAttempt = createTaskAttemptEntity(taskAttemptInfo);
+ taskAttempts.add(taskAttempt);
+ }
+ return taskAttempts;
+ }
+
+ private TimelineEntity createTaskAttemptEntity(TaskAttemptInfo taskAttemptInfo) {
+ TimelineEntity taskAttempt = new TimelineEntity();
+ taskAttempt.setType(TASK_ATTEMPT);
+ taskAttempt.setId(taskAttemptInfo.getAttemptId().toString());
+ taskAttempt.setCreatedTime(taskAttemptInfo.getStartTime());
+
+ taskAttempt.addInfo("START_TIME", taskAttemptInfo.getStartTime());
+ taskAttempt.addInfo("FINISH_TIME", taskAttemptInfo.getFinishTime());
+ taskAttempt.addInfo("MAP_FINISH_TIME",
+ taskAttemptInfo.getMapFinishTime());
+ taskAttempt.addInfo("SHUFFLE_FINISH_TIME",
+ taskAttemptInfo.getShuffleFinishTime());
+ taskAttempt.addInfo("SORT_FINISH_TIME",
+ taskAttemptInfo.getSortFinishTime());
+ taskAttempt.addInfo("TASK_STATUS", taskAttemptInfo.getTaskStatus());
+ taskAttempt.addInfo("STATE", taskAttemptInfo.getState());
+ taskAttempt.addInfo("ERROR", taskAttemptInfo.getError());
+ taskAttempt.addInfo("CONTAINER_ID",
+ taskAttemptInfo.getContainerId().toString());
+
+ // add metrics from counters
+ Counters counters = taskAttemptInfo.getCounters();
+ if (counters != null) {
+ addMetrics(taskAttempt, counters);
+ }
+ LOG.info("converted task attempt " + taskAttemptInfo.getAttemptId() +
+ " to a timeline entity");
+ return taskAttempt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8349b90c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
index 1c2e28d..f674ae1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
@@ -20,10 +20,7 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
import java.util.Date;
-import java.util.Random;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.IntWritable;
@@ -31,49 +28,35 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.SleepJob.SleepInputFormat;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
public class TimelineServicePerformanceV2 extends Configured implements Tool {
- private static final Log LOG =
- LogFactory.getLog(TimelineServicePerformanceV2.class);
-
static final int NUM_MAPS_DEFAULT = 1;
static final int SIMPLE_ENTITY_WRITER = 1;
- // constants for mtype = 1
- static final String KBS_SENT = "kbs sent";
- static final int KBS_SENT_DEFAULT = 1;
- static final String TEST_TIMES = "testtimes";
- static final int TEST_TIMES_DEFAULT = 100;
- static final String TIMELINE_SERVICE_PERFORMANCE_RUN_ID =
- "timeline.server.performance.run.id";
-
+ static final int JOB_HISTORY_FILE_REPLAY_MAPPER = 2;
static int mapperType = SIMPLE_ENTITY_WRITER;
protected static int printUsage() {
- // TODO is there a way to handle mapper-specific options more gracefully?
System.err.println(
"Usage: [-m <maps>] number of mappers (default: " + NUM_MAPS_DEFAULT +
")\n" +
- " [-mtype <mapper type in integer>] \n" +
+ " [-mtype <mapper type in integer>]\n" +
" 1. simple entity write mapper\n" +
- " [-s <(KBs)test>] number of KB per put (default: " +
- KBS_SENT_DEFAULT + " KB)\n" +
- " [-t] package sending iterations per mapper (default: " +
- TEST_TIMES_DEFAULT + ")\n");
+ " 2. job history file replay mapper\n" +
+ " [-s <(KBs)test>] number of KB per put (mtype=1, default: " +
+ SimpleEntityWriter.KBS_SENT_DEFAULT + " KB)\n" +
+ " [-t] package sending iterations per mapper (mtype=1, default: " +
+ SimpleEntityWriter.TEST_TIMES_DEFAULT + ")\n" +
+ " [-d <path>] root path of job history files (mtype=2)\n" +
+ " [-r <replay mode>] (mtype=2)\n" +
+ " 1. write all entities for a job in one put (default)\n" +
+ " 2. write one entity at a time\n");
GenericOptionsParser.printGenericCommandUsage(System.err);
return -1;
}
@@ -82,11 +65,9 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
* Configure a job given argv.
*/
public static boolean parseArgs(String[] args, Job job) throws IOException {
- // set the defaults
+ // set the common defaults
Configuration conf = job.getConfiguration();
conf.setInt(MRJobConfig.NUM_MAPS, NUM_MAPS_DEFAULT);
- conf.setInt(KBS_SENT, KBS_SENT_DEFAULT);
- conf.setInt(TEST_TIMES, TEST_TIMES_DEFAULT);
for (int i = 0; i < args.length; i++) {
if (args.length == i + 1) {
@@ -97,25 +78,24 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
if ("-m".equals(args[i])) {
if (Integer.parseInt(args[++i]) > 0) {
job.getConfiguration()
- .setInt(MRJobConfig.NUM_MAPS, (Integer.parseInt(args[i])));
+ .setInt(MRJobConfig.NUM_MAPS, Integer.parseInt(args[i]));
}
} else if ("-mtype".equals(args[i])) {
mapperType = Integer.parseInt(args[++i]);
- switch (mapperType) {
- case SIMPLE_ENTITY_WRITER:
- job.setMapperClass(SimpleEntityWriter.class);
- break;
- default:
- job.setMapperClass(SimpleEntityWriter.class);
- }
} else if ("-s".equals(args[i])) {
if (Integer.parseInt(args[++i]) > 0) {
- conf.setInt(KBS_SENT, (Integer.parseInt(args[i])));
+ conf.setInt(SimpleEntityWriter.KBS_SENT, Integer.parseInt(args[i]));
}
} else if ("-t".equals(args[i])) {
if (Integer.parseInt(args[++i]) > 0) {
- conf.setInt(TEST_TIMES, (Integer.parseInt(args[i])));
+ conf.setInt(SimpleEntityWriter.TEST_TIMES,
+ Integer.parseInt(args[i]));
}
+ } else if ("-d".equals(args[i])) {
+ conf.set(JobHistoryFileReplayMapper.PROCESSING_PATH, args[++i]);
+ } else if ("-r".equals(args[i])) {
+ conf.setInt(JobHistoryFileReplayMapper.REPLAY_MODE,
+ Integer.parseInt(args[++i]));
} else {
System.out.println("Unexpected argument: " + args[i]);
return printUsage() == 0;
@@ -128,6 +108,27 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
}
}
+ // handle mapper-specific settings
+ switch (mapperType) {
+ case JOB_HISTORY_FILE_REPLAY_MAPPER:
+ job.setMapperClass(JobHistoryFileReplayMapper.class);
+ String processingPath =
+ conf.get(JobHistoryFileReplayMapper.PROCESSING_PATH);
+ if (processingPath == null || processingPath.isEmpty()) {
+ System.out.println("processing path is missing while mtype = 2");
+ return printUsage() == 0;
+ }
+ break;
+ case SIMPLE_ENTITY_WRITER:
+ default:
+ job.setMapperClass(SimpleEntityWriter.class);
+ // use the current timestamp as the "run id" of the test: this will
+ // be used as simulating the cluster timestamp for apps
+ conf.setLong(SimpleEntityWriter.TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
+ System.currentTimeMillis());
+ break;
+ }
+
return true;
}
@@ -153,13 +154,6 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
return -1;
}
- // for mtype = 1
- // use the current timestamp as the "run id" of the test: this will be used
- // as simulating the cluster timestamp for apps
- Configuration conf = job.getConfiguration();
- conf.setLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
- System.currentTimeMillis());
-
Date startTime = new Date();
System.out.println("Job started: " + startTime);
int ret = job.waitForCompletion(true) ? 0 : 1;
@@ -172,7 +166,8 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).getValue();
double transacrate = writecounts * 1000 / (double)writetime;
double iorate = writesize * 1000 / (double)writetime;
- int numMaps = Integer.parseInt(conf.get(MRJobConfig.NUM_MAPS));
+ int numMaps =
+ Integer.parseInt(job.getConfiguration().get(MRJobConfig.NUM_MAPS));
System.out.println("TRANSACTION RATE (per mapper): " + transacrate +
" ops/s");
@@ -204,95 +199,31 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
'3', '4', '5', '6', '7', '8', '9', '0', ' ' };
/**
- * Adds simple entities with random string payload, events, metrics, and
- * configuration.
+ * Base mapper for writing entities to the timeline service. Subclasses
+ * override {@link #writeEntities(Configuration, TimelineCollectorManager,
+ * org.apache.hadoop.mapreduce.Mapper.Context)} to create and write entities
+ * to the timeline service.
*/
- public static class SimpleEntityWriter
+ public static abstract class EntityWriter
extends org.apache.hadoop.mapreduce.Mapper<IntWritable,IntWritable,Writable,Writable> {
+ @Override
public void map(IntWritable key, IntWritable val, Context context)
throws IOException {
- Configuration conf = context.getConfiguration();
- // simulate the app id with the task id
- int taskId = context.getTaskAttemptID().getTaskID().getId();
- long timestamp = conf.getLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID, 0);
- ApplicationId appId = ApplicationId.newInstance(timestamp, taskId);
-
- // create the app level timeline collector
+ // create the timeline collector manager wired with the writer
Configuration tlConf = new YarnConfiguration();
- AppLevelTimelineCollector collector =
- new AppLevelTimelineCollector(appId);
- collector.init(tlConf);
- collector.start();
-
+ TimelineCollectorManager manager = new TimelineCollectorManager("test");
+ manager.init(tlConf);
+ manager.start();
try {
- // set the context
- // flow id: job name, flow run id: timestamp, user id
- TimelineCollectorContext tlContext =
- collector.getTimelineEntityContext();
- tlContext.setFlowName(context.getJobName());
- tlContext.setFlowRunId(timestamp);
- tlContext.setUserId(context.getUser());
-
- final int kbs = Integer.parseInt(conf.get(KBS_SENT));
-
- long totalTime = 0;
- final int testtimes = Integer.parseInt(conf.get(TEST_TIMES));
- final Random rand = new Random();
- final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
- final char[] payLoad = new char[kbs * 1024];
-
- for (int i = 0; i < testtimes; i++) {
- // Generate a fixed length random payload
- for (int xx = 0; xx < kbs * 1024; xx++) {
- int alphaNumIdx = rand.nextInt(alphaNums.length);
- payLoad[xx] = alphaNums[alphaNumIdx];
- }
- String entId = taskAttemptId + "_" + Integer.toString(i);
- final TimelineEntity entity = new TimelineEntity();
- entity.setId(entId);
- entity.setType("FOO_ATTEMPT");
- entity.addInfo("PERF_TEST", payLoad);
- // add an event
- TimelineEvent event = new TimelineEvent();
- event.setTimestamp(System.currentTimeMillis());
- event.addInfo("foo_event", "test");
- entity.addEvent(event);
- // add a metric
- TimelineMetric metric = new TimelineMetric();
- metric.setId("foo_metric");
- metric.addValue(System.currentTimeMillis(), 123456789L);
- entity.addMetric(metric);
- // add a config
- entity.addConfig("foo", "bar");
-
- TimelineEntities entities = new TimelineEntities();
- entities.addEntity(entity);
- // use the current user for this purpose
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- long startWrite = System.nanoTime();
- try {
- collector.putEntities(entities, ugi);
- } catch (Exception e) {
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
- increment(1);
- e.printStackTrace();
- }
- long endWrite = System.nanoTime();
- totalTime += (endWrite-startWrite)/1000000L;
- }
- LOG.info("wrote " + testtimes + " entities (" + kbs*testtimes +
- " kB) in " + totalTime + " ms");
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
- increment(totalTime);
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
- increment(testtimes);
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).
- increment(kbs*testtimes);
+ // invoke the method to have the subclass write entities
+ writeEntities(tlConf, manager, context);
} finally {
- // clean up
- collector.close();
+ manager.close();
}
}
+
+ protected abstract void writeEntities(Configuration tlConf,
+ TimelineCollectorManager manager, Context context) throws IOException;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8349b90c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index e9f2085..165754d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -48,7 +48,7 @@ import com.google.common.annotations.VisibleForTesting;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public abstract class TimelineCollectorManager extends AbstractService {
+public class TimelineCollectorManager extends AbstractService {
private static final Log LOG =
LogFactory.getLog(TimelineCollectorManager.class);
@@ -90,10 +90,14 @@ public abstract class TimelineCollectorManager extends AbstractService {
Collections.synchronizedMap(
new HashMap<ApplicationId, TimelineCollector>());
- protected TimelineCollectorManager(String name) {
+ public TimelineCollectorManager(String name) {
super(name);
}
+ protected TimelineWriter getWriter() {
+ return writer;
+ }
+
/**
* Put the collector into the collection if an collector mapped by id does
* not exist.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8349b90c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index b22b39f..4385bbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -47,17 +47,17 @@ public class FileSystemTimelineWriterImpl extends AbstractService
private String outputRoot;
- /** Config param for timeline service storage tmp root for FILE YARN-3264 */
+ /** Config param for timeline service storage tmp root for FILE YARN-3264. */
public static final String TIMELINE_SERVICE_STORAGE_DIR_ROOT
- = YarnConfiguration.TIMELINE_SERVICE_PREFIX + "fs-writer.root-dir";
+ = YarnConfiguration.TIMELINE_SERVICE_PREFIX + "fs-writer.root-dir";
- /** default value for storage location on local disk */
+ /** default value for storage location on local disk. */
public static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
- = "/tmp/timeline_service_data";
+ = "/tmp/timeline_service_data";
public static final String ENTITIES_DIR = "entities";
- /** Default extension for output files */
+ /** Default extension for output files. */
public static final String TIMELINE_SERVICE_STORAGE_EXTENSION = ".thist";
FileSystemTimelineWriterImpl() {
@@ -81,9 +81,11 @@ public class FileSystemTimelineWriterImpl extends AbstractService
TimelineWriteResponse response) throws IOException {
PrintWriter out = null;
try {
- String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId,flowName,
- flowVersion, String.valueOf(flowRun), appId, entity.getType());
- String fileName = dir + entity.getId() + TIMELINE_SERVICE_STORAGE_EXTENSION;
+ String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId,
+ escape(flowName), escape(flowVersion), String.valueOf(flowRun), appId,
+ entity.getType());
+ String fileName = dir + entity.getId() +
+ TIMELINE_SERVICE_STORAGE_EXTENSION;
out =
new PrintWriter(new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(fileName, true), "UTF-8")));
@@ -145,4 +147,9 @@ public class FileSystemTimelineWriterImpl extends AbstractService
}
return path.toString();
}
+
+ // specifically escape the separator character
+ private static String escape(String str) {
+ return str.replace(File.separatorChar, '_');
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8349b90c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
new file mode 100644
index 0000000..f652ffd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
[41/50] [abbrv] hadoop git commit: YARN-3901. Populate flow run data
in the flow_run & flow activity tables (Vrushali C via sjlee)
Posted by sj...@apache.org.
YARN-3901. Populate flow run data in the flow_run & flow activity tables (Vrushali C via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78e76c7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78e76c7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78e76c7d
Branch: refs/heads/YARN-2928-rebase
Commit: 78e76c7d4ae6af5558ffddf21c42d62a6da5b861
Parents: 23fa8fa
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu Sep 17 10:34:52 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 17:05:03 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop-yarn-server-timelineservice/pom.xml | 13 +
.../storage/HBaseTimelineWriterImpl.java | 179 ++++++-
.../storage/TimelineSchemaCreator.java | 22 +-
.../storage/application/ApplicationColumn.java | 5 +-
.../application/ApplicationColumnPrefix.java | 15 +-
.../storage/apptoflow/AppToFlowColumn.java | 6 +-
.../timelineservice/storage/common/Column.java | 6 +-
.../storage/common/ColumnHelper.java | 93 +++-
.../storage/common/ColumnPrefix.java | 28 +-
.../storage/common/TimelineWriterUtils.java | 185 +++++++
.../storage/common/TimestampGenerator.java | 112 +++++
.../storage/common/package-info.java | 24 -
.../storage/entity/EntityColumn.java | 6 +-
.../storage/entity/EntityColumnPrefix.java | 20 +-
.../flow/AggregationCompactionDimension.java | 63 +++
.../storage/flow/AggregationOperation.java | 87 ++++
.../timelineservice/storage/flow/Attribute.java | 39 ++
.../storage/flow/FlowActivityColumnFamily.java | 54 +++
.../storage/flow/FlowActivityColumnPrefix.java | 243 ++++++++++
.../storage/flow/FlowActivityRowKey.java | 113 +++++
.../storage/flow/FlowActivityTable.java | 107 ++++
.../storage/flow/FlowRunColumn.java | 161 ++++++
.../storage/flow/FlowRunColumnFamily.java | 54 +++
.../storage/flow/FlowRunColumnPrefix.java | 239 +++++++++
.../storage/flow/FlowRunCoprocessor.java | 210 ++++++++
.../storage/flow/FlowRunRowKey.java | 50 ++
.../storage/flow/FlowRunTable.java | 141 ++++++
.../storage/flow/FlowScanner.java | 486 +++++++++++++++++++
.../storage/TestHBaseTimelineStorage.java | 28 +-
.../storage/flow/TestFlowDataGenerator.java | 213 ++++++++
.../flow/TestHBaseStorageFlowActivity.java | 372 ++++++++++++++
.../storage/flow/TestHBaseStorageFlowRun.java | 290 +++++++++++
33 files changed, 3562 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 302b325..ecbe9de 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -103,6 +103,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-4102. Add a "skip existing table" mode for timeline schema creator (Li
Lu via sjlee)
+ YARN-3901. Populate flow run data in the flow_run & flow activity tables
+ (Vrushali C via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index da7fadf..758feb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -174,6 +174,19 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <additionnalDependencies>
+ <additionnalDependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ </additionnalDependency>
+ </additionnalDependencies>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 772002d..7c4a5da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -33,11 +33,10 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
@@ -53,23 +52,36 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
/**
- * This implements a hbase based backend for storing application timeline entity
+ * This implements a hbase based backend for storing the timeline entity
* information.
+ * It writes to multiple tables at the backend
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class HBaseTimelineWriterImpl extends AbstractService implements
TimelineWriter {
+ private static final Log LOG = LogFactory
+ .getLog(HBaseTimelineWriterImpl.class);
+
private Connection conn;
private TypedBufferedMutator<EntityTable> entityTable;
private TypedBufferedMutator<AppToFlowTable> appToFlowTable;
private TypedBufferedMutator<ApplicationTable> applicationTable;
-
- private static final Log LOG = LogFactory
- .getLog(HBaseTimelineWriterImpl.class);
+ private TypedBufferedMutator<FlowActivityTable> flowActivityTable;
+ private TypedBufferedMutator<FlowRunTable> flowRunTable;
public HBaseTimelineWriterImpl() {
super(HBaseTimelineWriterImpl.class.getName());
@@ -91,6 +103,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
entityTable = new EntityTable().getTableMutator(hbaseConf, conn);
appToFlowTable = new AppToFlowTable().getTableMutator(hbaseConf, conn);
applicationTable = new ApplicationTable().getTableMutator(hbaseConf, conn);
+ flowRunTable = new FlowRunTable().getTableMutator(hbaseConf, conn);
+ flowActivityTable = new FlowActivityTable().getTableMutator(hbaseConf, conn);
}
/**
@@ -111,7 +125,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// if the entity is the application, the destination is the application
// table
- boolean isApplication = isApplicationEntity(te);
+ boolean isApplication = TimelineWriterUtils.isApplicationEntity(te);
byte[] rowKey = isApplication ?
ApplicationRowKey.getRowKey(clusterId, userId, flowName, flowRunId,
appId) :
@@ -124,37 +138,139 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
storeMetrics(rowKey, te.getMetrics(), isApplication);
storeRelations(rowKey, te, isApplication);
- if (isApplicationCreated(te)) {
- onApplicationCreated(
- clusterId, userId, flowName, flowVersion, flowRunId, appId, te);
+ if (isApplication) {
+ if (TimelineWriterUtils.isApplicationCreated(te)) {
+ onApplicationCreated(clusterId, userId, flowName, flowVersion,
+ flowRunId, appId, te);
+ }
+ // if it's an application entity, store metrics
+ storeFlowMetricsAppRunning(clusterId, userId, flowName, flowRunId,
+ appId, te);
+ // if application has finished, store it's finish time and write final
+ // values
+ // of all metrics
+ if (TimelineWriterUtils.isApplicationFinished(te)) {
+ onApplicationFinished(clusterId, userId, flowName, flowVersion,
+ flowRunId, appId, te);
+ }
}
}
return putStatus;
}
- private static boolean isApplicationEntity(TimelineEntity te) {
- return te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString());
+ private void onApplicationCreated(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRunId, String appId,
+ TimelineEntity te) throws IOException {
+ // store in App to flow table
+ storeInAppToFlowTable(clusterId, userId, flowName, flowVersion, flowRunId,
+ appId, te);
+ // store in flow run table
+ storeAppCreatedInFlowRunTable(clusterId, userId, flowName, flowVersion,
+ flowRunId, appId, te);
+ // store in flow activity table
+ storeInFlowActivityTable(clusterId, userId, flowName, flowVersion,
+ flowRunId, appId, te);
}
- private static boolean isApplicationCreated(TimelineEntity te) {
- if (isApplicationEntity(te)) {
- for (TimelineEvent event : te.getEvents()) {
- if (event.getId().equals(
- ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
- return true;
- }
- }
- }
- return false;
+ /*
+ * updates the {@link FlowActivityTable} with the Application TimelineEntity
+ * information
+ */
+ private void storeInFlowActivityTable(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRunId, String appId,
+ TimelineEntity te) throws IOException {
+ byte[] rowKey = FlowActivityRowKey.getRowKey(clusterId, userId, flowName);
+ byte[] qualifier = GenericObjectMapper.write(flowRunId);
+ FlowActivityColumnPrefix.RUN_ID.store(rowKey, flowActivityTable, qualifier,
+ null, flowVersion,
+ AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId));
}
- private void onApplicationCreated(String clusterId, String userId,
+ /*
+ * updates the {@link FlowRunTable} with Application Created information
+ */
+ private void storeAppCreatedInFlowRunTable(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRunId, String appId,
+ TimelineEntity te) throws IOException {
+ byte[] rowKey = FlowRunRowKey.getRowKey(clusterId, userId, flowName,
+ flowRunId);
+ FlowRunColumn.MIN_START_TIME.store(rowKey, flowRunTable, null,
+ te.getCreatedTime(),
+ AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId));
+ }
+
+ private void storeInAppToFlowTable(String clusterId, String userId,
String flowName, String flowVersion, long flowRunId, String appId,
TimelineEntity te) throws IOException {
byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
AppToFlowColumn.FLOW_ID.store(rowKey, appToFlowTable, null, flowName);
- AppToFlowColumn.FLOW_RUN_ID.store(
- rowKey, appToFlowTable, null, flowRunId);
+ AppToFlowColumn.FLOW_RUN_ID.store(rowKey, appToFlowTable, null, flowRunId);
+ }
+
+ /*
+ * updates the {@link FlowRunTable} and {@link FlowActivityTable} when an
+ * application has finished
+ */
+ private void onApplicationFinished(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRunId, String appId,
+ TimelineEntity te) throws IOException {
+ // store in flow run table
+ storeAppFinishedInFlowRunTable(clusterId, userId, flowName, flowRunId,
+ appId, te);
+
+ // indicate in the flow activity table that the app has finished
+ storeInFlowActivityTable(clusterId, userId, flowName, flowVersion,
+ flowRunId, appId, te);
+ }
+
+ /*
+ * Update the {@link FlowRunTable} with Application Finished information
+ */
+ private void storeAppFinishedInFlowRunTable(String clusterId, String userId,
+ String flowName, long flowRunId, String appId, TimelineEntity te)
+ throws IOException {
+ byte[] rowKey = FlowRunRowKey.getRowKey(clusterId, userId, flowName,
+ flowRunId);
+ Attribute attributeAppId = AggregationCompactionDimension.APPLICATION_ID
+ .getAttribute(appId);
+ FlowRunColumn.MAX_END_TIME.store(rowKey, flowRunTable, null,
+ TimelineWriterUtils.getApplicationFinishedTime(te), attributeAppId);
+
+ // store the final value of metrics since application has finished
+ Set<TimelineMetric> metrics = te.getMetrics();
+ if (metrics != null) {
+ storeFlowMetrics(rowKey, metrics, attributeAppId,
+ AggregationOperation.SUM_FINAL.getAttribute());
+ }
+ }
+
+ /*
+ * Updates the {@link FlowRunTable} with Application Metrics
+ */
+ private void storeFlowMetricsAppRunning(String clusterId, String userId,
+ String flowName, long flowRunId, String appId, TimelineEntity te)
+ throws IOException {
+ Set<TimelineMetric> metrics = te.getMetrics();
+ if (metrics != null) {
+ byte[] rowKey = FlowRunRowKey.getRowKey(clusterId, userId, flowName,
+ flowRunId);
+ storeFlowMetrics(rowKey, metrics,
+ AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId));
+ }
+ }
+
+ private void storeFlowMetrics(byte[] rowKey, Set<TimelineMetric> metrics,
+ Attribute... attributes) throws IOException {
+ for (TimelineMetric metric : metrics) {
+ String metricColumnQualifier = metric.getId();
+ Map<Long, Number> timeseries = metric.getValues();
+ for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
+ Long timestamp = timeseriesEntry.getKey();
+ FlowRunColumnPrefix.METRIC.store(rowKey, flowRunTable,
+ metricColumnQualifier, timestamp, timeseriesEntry.getValue(),
+ attributes);
+ }
+ }
}
private void storeRelations(byte[] rowKey, TimelineEntity te,
@@ -184,7 +300,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// id3?id4?id5
String compoundValue =
Separator.VALUES.joinEncoded(connectedEntity.getValue());
-
columnPrefix.store(rowKey, table, connectedEntity.getKey(), null,
compoundValue);
}
@@ -342,6 +457,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
entityTable.flush();
appToFlowTable.flush();
applicationTable.flush();
+ flowRunTable.flush();
+ flowActivityTable.flush();
}
/**
@@ -364,6 +481,16 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
LOG.info("closing the application table");
applicationTable.close();
}
+ if (flowRunTable != null) {
+ LOG.info("closing the flow run table");
+ // The close API performs flushing and releases any resources held
+ flowRunTable.close();
+ }
+ if (flowActivityTable != null) {
+ LOG.info("closing the flowActivityTable table");
+ // The close API performs flushing and releases any resources held
+ flowActivityTable.close();
+ }
if (conn != null) {
LOG.info("closing the hbase Connection");
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index e7e51a7..cbcff4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -42,6 +42,8 @@ import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
/**
* This creates the schema for a hbase based backend for storing application
@@ -199,7 +201,7 @@ public class TimelineSchemaCreator {
return commandLine;
}
- private static void createAllTables(Configuration hbaseConf,
+ public static void createAllTables(Configuration hbaseConf,
boolean skipExisting) throws IOException {
Connection conn = null;
@@ -236,6 +238,24 @@ public class TimelineSchemaCreator {
throw e;
}
}
+ try {
+ new FlowRunTable().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
+ try {
+ new FlowActivityTable().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
} finally {
if (conn != null) {
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
index c028386..802626d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
/**
* Identifies fully qualified columns for the {@link ApplicationTable}.
@@ -76,9 +77,9 @@ public enum ApplicationColumn implements Column<ApplicationTable> {
public void store(byte[] rowKey,
TypedBufferedMutator<ApplicationTable> tableMutator, Long timestamp,
- Object inputValue) throws IOException {
+ Object inputValue, Attribute... attributes) throws IOException {
column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
- inputValue);
+ inputValue, attributes);
}
public Object readResult(Result result) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
index ad1def6..d7b5773 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
/**
* Identifies partially qualified columns for the application table.
@@ -112,7 +113,8 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
*/
public void store(byte[] rowKey,
TypedBufferedMutator<ApplicationTable> tableMutator, byte[] qualifier,
- Long timestamp, Object inputValue) throws IOException {
+ Long timestamp, Object inputValue, Attribute... attributes)
+ throws IOException {
// Null check
if (qualifier == null) {
@@ -123,8 +125,9 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
byte[] columnQualifier =
ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
- }
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
+ attributes);
+ }
/*
* (non-Javadoc)
@@ -137,7 +140,8 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
*/
public void store(byte[] rowKey,
TypedBufferedMutator<ApplicationTable> tableMutator, String qualifier,
- Long timestamp, Object inputValue) throws IOException {
+ Long timestamp, Object inputValue, Attribute...attributes)
+ throws IOException {
// Null check
if (qualifier == null) {
@@ -148,7 +152,8 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
byte[] columnQualifier =
ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
+ attributes);
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
index 423037a..859fdca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
@@ -25,8 +25,10 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
import java.io.IOException;
+import java.util.Map;
/**
* Identifies fully qualified columns for the {@link AppToFlowTable}.
@@ -67,9 +69,9 @@ public enum AppToFlowColumn implements Column<AppToFlowTable> {
public void store(byte[] rowKey,
TypedBufferedMutator<AppToFlowTable> tableMutator, Long timestamp,
- Object inputValue) throws IOException {
+ Object inputValue, Attribute... attributes) throws IOException {
column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
- inputValue);
+ inputValue, attributes);
}
public Object readResult(Result result) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
index 3397d62..64c1cda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
/**
* A Column represents the way to store a fully qualified column in a specific
@@ -38,12 +39,15 @@ public interface Column<T> {
* column.
* @param timestamp version timestamp. When null the server timestamp will be
* used.
+ * @param attributes Map of attributes for this mutation. used in the coprocessor
+ * to set/read the cell tags. Can be null.
* @param inputValue the value to write to the rowKey and column qualifier.
* Nothing gets written when null.
* @throws IOException
*/
public void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
- Long timestamp, Object inputValue) throws IOException;
+ Long timestamp, Object inputValue, Attribute... attributes)
+ throws IOException;
/**
* Get the latest version of this specified column. Note: this call clones the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
index f1b7c58..3a2e088 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
/**
* This class is meant to be used only by explicit Columns, and not directly to
* write by clients.
@@ -58,31 +59,66 @@ public class ColumnHelper<T> {
* Sends a Mutation to the table. The mutations will be buffered and sent over
* the wire as part of a batch.
*
- * @param rowKey identifying the row to write. Nothing gets written when null.
- * @param tableMutator used to modify the underlying HBase table
- * @param columnQualifier column qualifier. Nothing gets written when null.
- * @param timestamp version timestamp. When null the server timestamp will be
- * used.
- * @param inputValue the value to write to the rowKey and column qualifier.
- * Nothing gets written when null.
+ * @param rowKey
+ * identifying the row to write. Nothing gets written when null.
+ * @param tableMutator
+ * used to modify the underlying HBase table
+ * @param columnQualifier
+ * column qualifier. Nothing gets written when null.
+ * @param timestamp
+ * version timestamp. When null the current timestamp multiplied with
+ * TimestampGenerator.TS_MULTIPLIER and added with last 3 digits of
+ * app id will be used
+ * @param inputValue
+ * the value to write to the rowKey and column qualifier. Nothing
+ * gets written when null.
* @throws IOException
*/
public void store(byte[] rowKey, TypedBufferedMutator<?> tableMutator,
- byte[] columnQualifier, Long timestamp, Object inputValue)
- throws IOException {
+ byte[] columnQualifier, Long timestamp, Object inputValue,
+ Attribute... attributes) throws IOException {
if ((rowKey == null) || (columnQualifier == null) || (inputValue == null)) {
return;
}
Put p = new Put(rowKey);
+ timestamp = getPutTimestamp(timestamp, attributes);
+ p.addColumn(columnFamilyBytes, columnQualifier, timestamp,
+ GenericObjectMapper.write(inputValue));
+ if ((attributes != null) && (attributes.length > 0)) {
+ for (Attribute attribute : attributes) {
+ p.setAttribute(attribute.getName(), attribute.getValue());
+ }
+ }
+ tableMutator.mutate(p);
+ }
+ /*
+ * Figures out the cell timestamp used in the Put For storing into flow run
+ * table. We would like to left shift the timestamp and supplement it with the
+ * AppId id so that there are no collisions in the flow run table's cells
+ */
+ private long getPutTimestamp(Long timestamp, Attribute[] attributes) {
if (timestamp == null) {
- p.addColumn(columnFamilyBytes, columnQualifier,
- GenericObjectMapper.write(inputValue));
- } else {
- p.addColumn(columnFamilyBytes, columnQualifier, timestamp,
- GenericObjectMapper.write(inputValue));
+ timestamp = System.currentTimeMillis();
}
- tableMutator.mutate(p);
+ String appId = getAppIdFromAttributes(attributes);
+ long supplementedTS = TimestampGenerator.getSupplementedTimestamp(
+ timestamp, appId);
+ return supplementedTS;
+ }
+
+ private String getAppIdFromAttributes(Attribute[] attributes) {
+ if (attributes == null) {
+ return null;
+ }
+ String appId = null;
+ for (Attribute attribute : attributes) {
+ if (AggregationCompactionDimension.APPLICATION_ID.toString().equals(
+ attribute.getName())) {
+ appId = Bytes.toString(attribute.getValue());
+ }
+ }
+ return appId;
}
/**
@@ -171,7 +207,9 @@ public class ColumnHelper<T> {
for (Entry<Long, byte[]> cell : cells.entrySet()) {
V value =
(V) GenericObjectMapper.read(cell.getValue());
- cellResults.put(cell.getKey(), value);
+ cellResults.put(
+ TimestampGenerator.getTruncatedTimestamp(cell.getKey()),
+ value);
}
}
results.put(columnName, cellResults);
@@ -315,6 +353,27 @@ public class ColumnHelper<T> {
/**
* @param columnPrefixBytes The byte representation for the column prefix.
* Should not contain {@link Separator#QUALIFIERS}.
+ * @param qualifier for the remainder of the column.
+ * @return fully sanitized column qualifier that is a combination of prefix
+ * and qualifier. If prefix is null, the result is simply the encoded
+ * qualifier without any separator.
+ */
+ public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
+ long qualifier) {
+
+ if (columnPrefixBytes == null) {
+ return Bytes.toBytes(qualifier);
+ }
+
+ // Convert qualifier to lower case, strip of separators and tag on column
+ // prefix.
+ byte[] columnQualifier =
+ Separator.QUALIFIERS.join(columnPrefixBytes, Bytes.toBytes(qualifier));
+ return columnQualifier;
+ }
+ /**
+ * @param columnPrefixBytes The byte representation for the column prefix.
+ * Should not contain {@link Separator#QUALIFIERS}.
* @param qualifier the byte representation for the remainder of the column.
* @return fully sanitized column qualifier that is a combination of prefix
* and qualifier. If prefix is null, the result is simply the encoded
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
index 509ff49..db49098 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
@@ -23,6 +23,7 @@ import java.util.NavigableMap;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
/**
* Used to represent a partially qualified column, where the actual column name
@@ -43,12 +44,36 @@ public interface ColumnPrefix<T> {
* @param qualifier column qualifier. Nothing gets written when null.
* @param timestamp version timestamp. When null the server timestamp will be
* used.
+ *@param attributes attributes for the mutation that are used by the coprocessor
+ * to set/read the cell tags
* @param inputValue the value to write to the rowKey and column qualifier.
* Nothing gets written when null.
* @throws IOException
*/
public void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
- String qualifier, Long timestamp, Object inputValue) throws IOException;
+ byte[] qualifier, Long timestamp, Object inputValue,
+ Attribute... attributes) throws IOException;
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey identifying the row to write. Nothing gets written when null.
+ * @param tableMutator used to modify the underlying HBase table. Caller is
+ * responsible to pass a mutator for the table that actually has this
+ * column.
+ * @param qualifier column qualifier. Nothing gets written when null.
+ * @param timestamp version timestamp. When null the server timestamp will be
+ * used.
+ *@param attributes attributes for the mutation that are used by the coprocessor
+ * to set/read the cell tags
+ * @param inputValue the value to write to the rowKey and column qualifier.
+ * Nothing gets written when null.
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
+ String qualifier, Long timestamp, Object inputValue,
+ Attribute... attributes) throws IOException;
/**
* Get the latest version of this specified column. Note: this call clones the
@@ -81,4 +106,5 @@ public interface ColumnPrefix<T> {
*/
public <V> NavigableMap<String, NavigableMap<Long, V>>
readResultsWithTimestamps(Result result) throws IOException;
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
index 58bdedc7e..371371a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
@@ -19,9 +19,19 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.common;
import java.util.ArrayList;
import java.util.List;
+import java.util.SortedSet;
+import java.util.Map.Entry;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
/**
* bunch of utility functions used across TimelineWriter classes
@@ -36,6 +46,9 @@ public class TimelineWriterUtils {
/** indicator for no limits for splitting */
public static final int NO_LIMIT_SPLIT = -1;
+ /** milliseconds in one day */
+ public static final long MILLIS_ONE_DAY = 86400000L;
+
/**
* Splits the source array into multiple array segments using the given
* separator, up to a maximum of count items. This will naturally produce
@@ -140,4 +153,176 @@ public class TimelineWriterUtils {
return Long.MAX_VALUE - key;
}
+ /**
+ * returns the timestamp of that day's start (which is midnight 00:00:00 AM)
+ * for a given input timestamp
+ *
+ * @param ts
+ * @return timestamp of that day's beginning (midnight)
+ */
+ public static long getTopOfTheDayTimestamp(long ts) {
+ long dayTimestamp = ts - (ts % MILLIS_ONE_DAY);
+ return dayTimestamp;
+ }
+
+ /**
+ * Combines the input array of attributes and the input aggregation operation
+ * into a new array of attributes.
+ *
+ * @param attributes
+ * @param aggOp
+ * @return array of combined attributes
+ */
+ public static Attribute[] combineAttributes(Attribute[] attributes,
+ AggregationOperation aggOp) {
+ int newLength = getNewLengthCombinedAttributes(attributes, aggOp);
+ Attribute[] combinedAttributes = new Attribute[newLength];
+
+ if (attributes != null) {
+ System.arraycopy(attributes, 0, combinedAttributes, 0, attributes.length);
+ }
+
+ if (aggOp != null) {
+ Attribute a2 = aggOp.getAttribute();
+ combinedAttributes[newLength - 1] = a2;
+ }
+ return combinedAttributes;
+ }
+
+ /**
+ * Returns a number for the new array size. The new array is the combination
+ * of input array of attributes and the input aggregation operation.
+ *
+ * @param attributes
+ * @param aggOp
+ * @return the size for the new array
+ */
+ private static int getNewLengthCombinedAttributes(Attribute[] attributes,
+ AggregationOperation aggOp) {
+ int oldLength = getAttributesLength(attributes);
+ int aggLength = getAppOpLength(aggOp);
+ return oldLength + aggLength;
+ }
+
+ private static int getAppOpLength(AggregationOperation aggOp) {
+ if (aggOp != null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ private static int getAttributesLength(Attribute[] attributes) {
+ if (attributes != null) {
+ return attributes.length;
+ }
+ return 0;
+ }
+
+ /**
+ * checks if an application has finished
+ *
+ * @param te
+ * @return true if application has finished else false
+ */
+ public static boolean isApplicationFinished(TimelineEntity te) {
+ SortedSet<TimelineEvent> allEvents = te.getEvents();
+ if ((allEvents != null) && (allEvents.size() > 0)) {
+ TimelineEvent event = allEvents.last();
+ if (event.getId().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * get the time at which an app finished
+ *
+ * @param te
+ * @return true if application has finished else false
+ */
+ public static long getApplicationFinishedTime(TimelineEntity te) {
+ SortedSet<TimelineEvent> allEvents = te.getEvents();
+ if ((allEvents != null) && (allEvents.size() > 0)) {
+ TimelineEvent event = allEvents.last();
+ if (event.getId().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
+ return event.getTimestamp();
+ }
+ }
+ return 0l;
+ }
+
+ /**
+ * Checks if the input TimelineEntity object is an ApplicationEntity.
+ *
+ * @param te
+ * @return true if input is an ApplicationEntity, false otherwise
+ */
+ public static boolean isApplicationEntity(TimelineEntity te) {
+ return te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString());
+ }
+
+ /**
+ * Checks for the APPLICATION_CREATED event.
+ *
+ * @param te
+ * @return true is application event exists, false otherwise
+ */
+ public static boolean isApplicationCreated(TimelineEntity te) {
+ if (isApplicationEntity(te)) {
+ for (TimelineEvent event : te.getEvents()) {
+ if (event.getId()
+ .equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns the first seen aggregation operation as seen in the list of input
+ * tags or null otherwise
+ *
+ * @param tags
+ * @return AggregationOperation
+ */
+ public static AggregationOperation getAggregationOperationFromTagsList(
+ List<Tag> tags) {
+ for (AggregationOperation aggOp : AggregationOperation.values()) {
+ for (Tag tag : tags) {
+ if (tag.getType() == aggOp.getTagType()) {
+ return aggOp;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Creates a {@link Tag} from the input attribute.
+ *
+ * @param attribute
+ * @return Tag
+ */
+ public static Tag getTagFromAttribute(Entry<String, byte[]> attribute) {
+ // attribute could be either an Aggregation Operation or
+ // an Aggregation Dimension
+ // Get the Tag type from either
+ AggregationOperation aggOp = AggregationOperation
+ .getAggregationOperation(attribute.getKey());
+ if (aggOp != null) {
+ Tag t = new Tag(aggOp.getTagType(), attribute.getValue());
+ return t;
+ }
+
+ AggregationCompactionDimension aggCompactDim = AggregationCompactionDimension
+ .getAggregationCompactionDimension(attribute.getKey());
+ if (aggCompactDim != null) {
+ Tag t = new Tag(aggCompactDim.getTagType(), attribute.getValue());
+ return t;
+ }
+ return null;
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
new file mode 100644
index 0000000..555b64e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+/**
+ * Utility class that allows HBase coprocessors to interact with unique
+ * timestamps.
+ */
+public class TimestampGenerator {
+
+ /*
+ * if this is changed, then reading cell timestamps written with older
+ * multiplier value will not work
+ */
+ public static final long TS_MULTIPLIER = 1000L;
+
+ private final AtomicLong lastTimestamp = new AtomicLong();
+
+ /**
+ * Returns the current wall clock time in milliseconds, multiplied by the
+ * required precision.
+ */
+ public long currentTime() {
+ // We want to align cell timestamps with current time.
+ // cell timestamps are not be less than
+ // System.currentTimeMillis() * TS_MULTIPLIER.
+ return System.currentTimeMillis() * TS_MULTIPLIER;
+ }
+
+ /**
+ * Returns a timestamp value unique within the scope of this
+ * {@code TimestampGenerator} instance. For usage by HBase
+ * {@code RegionObserver} coprocessors, this normally means unique within a
+ * given region.
+ *
+ * Unlikely scenario of generating a non-unique timestamp: if there is a
+ * sustained rate of more than 1M hbase writes per second AND if region fails
+ * over within that time range of timestamps being generated then there may be
+ * collisions writing to a cell version of the same column.
+ */
+ public long getUniqueTimestamp() {
+ long lastTs;
+ long nextTs;
+ do {
+ lastTs = lastTimestamp.get();
+ nextTs = Math.max(lastTs + 1, currentTime());
+ } while (!lastTimestamp.compareAndSet(lastTs, nextTs));
+ return nextTs;
+ }
+
+ /**
+ * returns a timestamp multiplied with TS_MULTIPLIER and last few digits of
+ * application id
+ *
+ * Unlikely scenario of generating a timestamp that is a duplicate: If more
+ * than a 1000 concurrent apps are running in one flow run AND write to same
+ * column at the same time, then say appId of 1001 will overlap with appId of
+ * 001 and there may be collisions for that flow run's specific column.
+ *
+ * @param incomingTS
+ * @param appId
+ * @return a timestamp multiplied with TS_MULTIPLIER and last few digits of
+ * application id
+ */
+ public static long getSupplementedTimestamp(long incomingTS, String appId) {
+ long suffix = getAppIdSuffix(appId);
+ long outgoingTS = incomingTS * TS_MULTIPLIER + suffix;
+ return outgoingTS;
+
+ }
+
+ private static long getAppIdSuffix(String appIdStr) {
+ if (appIdStr == null) {
+ return 0L;
+ }
+ ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
+ long id = appId.getId() % TS_MULTIPLIER;
+ return id;
+ }
+
+ /**
+ * truncates the last few digits of the timestamp which were supplemented by
+ * the TimestampGenerator#getSupplementedTimestamp function
+ *
+ * @param incomingTS
+ * @return a truncated timestamp value
+ */
+ public static long getTruncatedTimestamp(long incomingTS) {
+ return incomingTS / TS_MULTIPLIER;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
deleted file mode 100644
index 32577fb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
index 26e7748..8ae19b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
import java.io.IOException;
+import java.util.Map;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
@@ -26,6 +27,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
/**
* Identifies fully qualified columns for the {@link EntityTable}.
@@ -81,9 +83,9 @@ public enum EntityColumn implements Column<EntityTable> {
public void store(byte[] rowKey,
TypedBufferedMutator<EntityTable> tableMutator, Long timestamp,
- Object inputValue) throws IOException {
+ Object inputValue, Attribute... attributes) throws IOException {
column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
- inputValue);
+ inputValue, attributes);
}
public Object readResult(Result result) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index 75ff742..0d4e5a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
/**
* Identifies partially qualified columns for the entity table.
@@ -108,11 +109,13 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
* org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
* #store(byte[],
* org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object,
+ * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[])
*/
public void store(byte[] rowKey,
TypedBufferedMutator<EntityTable> tableMutator, String qualifier,
- Long timestamp, Object inputValue) throws IOException {
+ Long timestamp, Object inputValue, Attribute... attributes)
+ throws IOException {
// Null check
if (qualifier == null) {
@@ -123,8 +126,9 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
byte[] columnQualifier =
ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
- }
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
+ attributes);
+ }
/*
* (non-Javadoc)
@@ -137,7 +141,8 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
*/
public void store(byte[] rowKey,
TypedBufferedMutator<EntityTable> tableMutator, byte[] qualifier,
- Long timestamp, Object inputValue) throws IOException {
+ Long timestamp, Object inputValue, Attribute... attributes)
+ throws IOException {
// Null check
if (qualifier == null) {
@@ -148,8 +153,9 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
byte[] columnQualifier =
ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
- }
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
+ attributes);
+ }
/*
* (non-Javadoc)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
new file mode 100644
index 0000000..ff12c7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Identifies the compaction dimensions for the data in the {@link FlowRunTable}
+ * .
+ */
+public enum AggregationCompactionDimension {
+
+ /**
+ * the application id
+ */
+ APPLICATION_ID((byte) 101);
+
+ private byte tagType;
+ private byte[] inBytes;
+
+ private AggregationCompactionDimension(byte tagType) {
+ this.tagType = tagType;
+ this.inBytes = Bytes.toBytes(this.name());
+ }
+
+ public Attribute getAttribute(String attributeValue) {
+ return new Attribute(this.name(), Bytes.toBytes(attributeValue));
+ }
+
+ public byte getTagType() {
+ return tagType;
+ }
+
+ public byte[] getInBytes() {
+ return this.inBytes.clone();
+ }
+
+ public static AggregationCompactionDimension getAggregationCompactionDimension(
+ String aggCompactDimStr) {
+ for (AggregationCompactionDimension aggDim : AggregationCompactionDimension
+ .values()) {
+ if (aggDim.name().equals(aggCompactDimStr)) {
+ return aggDim;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
new file mode 100644
index 0000000..c635ce6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Identifies the attributes to be set for puts into the {@link FlowRunTable}.
+ * The numbers used for tagType are prime numbers
+ */
+public enum AggregationOperation {
+
+ /**
+ * When the flow was started.
+ */
+ MIN((byte) 71),
+
+ /**
+ * When it ended.
+ */
+ MAX((byte) 73),
+
+ /**
+ * The metrics of the flow
+ */
+ SUM((byte) 79),
+
+ /**
+ * application running
+ */
+ SUM_FINAL((byte) 83),
+
+ /**
+ * compact
+ */
+ COMPACT((byte) 89);
+
+ private byte tagType;
+ private byte[] inBytes;
+
+ private AggregationOperation(byte tagType) {
+ this.tagType = tagType;
+ this.inBytes = Bytes.toBytes(this.name());
+ }
+
+ public Attribute getAttribute() {
+ return new Attribute(this.name(), this.inBytes);
+ }
+
+ public byte getTagType() {
+ return tagType;
+ }
+
+ public byte[] getInBytes() {
+ return this.inBytes.clone();
+ }
+
+ /**
+ * returns the AggregationOperation enum that represents that string
+ * @param aggOpStr
+ * @return the AggregationOperation enum that represents that string
+ */
+ public static AggregationOperation getAggregationOperation(String aggOpStr) {
+ for (AggregationOperation aggOp : AggregationOperation.values()) {
+ if (aggOp.name().equals(aggOpStr)) {
+ return aggOp;
+ }
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java
new file mode 100644
index 0000000..d3de518
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+/**
+ * Defines the attribute tuple to be set for puts into the {@link FlowRunTable}.
+ */
+public class Attribute {
+ private final String name;
+ private final byte[] value;
+
+ public Attribute(String name, byte[] value) {
+ this.name = name;
+ this.value = value.clone();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public byte[] getValue() {
+ return value.clone();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
new file mode 100644
index 0000000..d991b42
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the flow run table column families.
+ */
+public enum FlowActivityColumnFamily implements ColumnFamily<FlowActivityTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value
+ * create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ private FlowActivityColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
new file mode 100644
index 0000000..b899e5c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies partially qualified columns for the {@link FlowActivityTable}
+ */
+public enum FlowActivityColumnPrefix implements ColumnPrefix<FlowActivityTable> {
+
+ /**
+ * To store run ids of the flows
+ */
+ RUN_ID(FlowActivityColumnFamily.INFO, "r", null);
+
+ private final ColumnHelper<FlowActivityTable> column;
+ private final ColumnFamily<FlowActivityTable> columnFamily;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+
+ private final AggregationOperation aggOp;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily
+ * that this column is stored in.
+ * @param columnPrefix
+ * for this column.
+ */
+ private FlowActivityColumnPrefix(
+ ColumnFamily<FlowActivityTable> columnFamily, String columnPrefix,
+ AggregationOperation aggOp) {
+ column = new ColumnHelper<FlowActivityTable>(columnFamily);
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes = Bytes.toBytes(Separator.SPACE
+ .encode(columnPrefix));
+ }
+ this.aggOp = aggOp;
+ }
+
+ /**
+ * @return the column name value
+ */
+ public String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ public byte[] getColumnPrefixBytes() {
+ return columnPrefixBytes.clone();
+ }
+
+ public AggregationOperation getAttribute() {
+ return aggOp;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, byte[], java.lang.Long, java.lang.Object,
+ * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[])
+ */
+ @Override
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<FlowActivityTable> tableMutator, byte[] qualifier,
+ Long timestamp, Object inputValue, Attribute... attributes)
+ throws IOException {
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier = ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifier);
+ Attribute[] combinedAttributes = TimelineWriterUtils.combineAttributes(
+ attributes, this.aggOp);
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
+ combinedAttributes);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
+ */
+ public Object readResult(Result result, String qualifier) throws IOException {
+ byte[] columnQualifier = ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifier);
+ return column.readResult(result, columnQualifier);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResults(org.apache.hadoop.hbase.client.Result)
+ */
+ public Map<String, Object> readResults(Result result) throws IOException {
+ return column.readResults(result, columnPrefixBytes);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
+ */
+ public <T> NavigableMap<String, NavigableMap<Long, T>> readResultsWithTimestamps(
+ Result result) throws IOException {
+ return column.readResultsWithTimestamps(result, columnPrefixBytes);
+ }
+
+ /**
+ * Retrieve an {@link FlowActivityColumnPrefix} given a name, or null if there
+ * is no match. The following holds true: {@code columnFor(x) == columnFor(y)}
+ * if and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnPrefix
+ * Name of the column to retrieve
+ * @return the corresponding {@link FlowActivityColumnPrefix} or null
+ */
+ public static final FlowActivityColumnPrefix columnFor(String columnPrefix) {
+
+ // Match column based on value, assume column family matches.
+ for (FlowActivityColumnPrefix flowActivityColPrefix : FlowActivityColumnPrefix
+ .values()) {
+ // Find a match based only on name.
+ if (flowActivityColPrefix.getColumnPrefix().equals(columnPrefix)) {
+ return flowActivityColPrefix;
+ }
+ }
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link FlowActivityColumnPrefix} given a name, or null if there
+ * is no match. The following holds true:
+ * {@code columnFor(a,x) == columnFor(b,y)} if and only if
+ * {@code (x == y == null)} or {@code a.equals(b) & x.equals(y)}
+ *
+ * @param columnFamily
+ * The columnFamily for which to retrieve the column.
+ * @param columnPrefix
+ * Name of the column to retrieve
+ * @return the corresponding {@link FlowActivityColumnPrefix} or null if both
+ * arguments don't match.
+ */
+ public static final FlowActivityColumnPrefix columnFor(
+ FlowActivityColumnFamily columnFamily, String columnPrefix) {
+
+ // TODO: needs unit test to confirm and need to update javadoc to explain
+ // null prefix case.
+
+ for (FlowActivityColumnPrefix flowActivityColumnPrefix : FlowActivityColumnPrefix
+ .values()) {
+ // Find a match based column family and on name.
+ if (flowActivityColumnPrefix.columnFamily.equals(columnFamily)
+ && (((columnPrefix == null) && (flowActivityColumnPrefix
+ .getColumnPrefix() == null)) || (flowActivityColumnPrefix
+ .getColumnPrefix().equals(columnPrefix)))) {
+ return flowActivityColumnPrefix;
+ }
+ }
+ // Default to null
+ return null;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object,
+ * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[])
+ */
+ @Override
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<FlowActivityTable> tableMutator, String qualifier,
+ Long timestamp, Object inputValue, Attribute... attributes)
+ throws IOException {
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier = ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifier);
+ Attribute[] combinedAttributes = TimelineWriterUtils.combineAttributes(
+ attributes, this.aggOp);
+ column.store(rowKey, tableMutator, columnQualifier, null, inputValue,
+ combinedAttributes);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
new file mode 100644
index 0000000..19e4e83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+
+/**
+ * Represents a rowkey for the flow activity table.
+ */
+public class FlowActivityRowKey {
+
+ private final String clusterId;
+ private final long dayTs;
+ private final String userId;
+ private final String flowId;
+
+ public FlowActivityRowKey(String clusterId, long dayTs, String userId,
+ String flowId) {
+ this.clusterId = clusterId;
+ this.dayTs = dayTs;
+ this.userId = userId;
+ this.flowId = flowId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public long getDayTimestamp() {
+ return dayTs;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public String getFlowId() {
+ return flowId;
+ }
+
+ /**
+ * Constructs a row key for the flow activity table as follows:
+ * {@code clusterId!dayTimestamp!user!flowId}
+ *
+ * Will insert into current day's record in the table
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @return byte array with the row key prefix
+ */
+ public static byte[] getRowKey(String clusterId, String userId, String flowId) {
+ long dayTs = TimelineWriterUtils.getTopOfTheDayTimestamp(System
+ .currentTimeMillis());
+ return getRowKey(clusterId, dayTs, userId, flowId);
+ }
+
+ /**
+ * Constructs a row key for the flow activity table as follows:
+ * {@code clusterId!dayTimestamp!user!flowId}
+ *
+ * @param clusterId
+ * @param dayTs
+ * @param userId
+ * @param flowId
+ * @return byte array for the row key
+ */
+ public static byte[] getRowKey(String clusterId, long dayTs, String userId,
+ String flowId) {
+ return Separator.QUALIFIERS.join(
+ Bytes.toBytes(Separator.QUALIFIERS.encode(clusterId)),
+ Bytes.toBytes(TimelineWriterUtils.invert(dayTs)),
+ Bytes.toBytes(Separator.QUALIFIERS.encode(userId)),
+ Bytes.toBytes(Separator.QUALIFIERS.encode(flowId)));
+ }
+
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ */
+ public static FlowActivityRowKey parseRowKey(byte[] rowKey) {
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
+
+ if (rowKeyComponents.length < 4) {
+ throw new IllegalArgumentException("the row key is not valid for "
+ + "a flow activity");
+ }
+
+ String clusterId = Separator.QUALIFIERS.decode(Bytes
+ .toString(rowKeyComponents[0]));
+ long dayTs = TimelineWriterUtils.invert(Bytes.toLong(rowKeyComponents[1]));
+ String userId = Separator.QUALIFIERS.decode(Bytes
+ .toString(rowKeyComponents[2]));
+ String flowId = Separator.QUALIFIERS.decode(Bytes
+ .toString(rowKeyComponents[3]));
+ return new FlowActivityRowKey(clusterId, dayTs, userId, flowId);
+ }
+}
[24/50] [abbrv] hadoop git commit: YARN-3993. Changed to use the AM
flag in ContainerContext determine AM container in
TestPerNodeTimelineCollectorsAuxService. Contributed by Sunil G.
Posted by sj...@apache.org.
YARN-3993. Changed to use the AM flag in ContainerContext determine AM container in TestPerNodeTimelineCollectorsAuxService. Contributed by Sunil G.
(cherry picked from commit 9e48f9ff2ce08f3dcdd8d60bacb697664b92196f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb57d79c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb57d79c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb57d79c
Branch: refs/heads/YARN-2928-rebase
Commit: fb57d79cd4b5f78f3c47adfef61b328f60dfb978
Parents: eb4da07
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Aug 3 16:55:44 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:40 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../collector/PerNodeTimelineCollectorsAuxService.java | 13 +++----------
.../TestPerNodeTimelineCollectorsAuxService.java | 5 +++++
3 files changed, 11 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb57d79c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e71c8f1..b854cca 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -118,6 +118,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3908. Fixed bugs in HBaseTimelineWriterImpl. (Vrushali C and Sangjin
Lee via zjshen)
+ YARN-3993. Changed to use the AM flag in ContainerContext determine AM
+ container in TestPerNodeTimelineCollectorsAuxService. (Sunil G via zjshen)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb57d79c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 3ede97a..befaa83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.api.AuxiliaryService;
import org.apache.hadoop.yarn.server.api.ContainerContext;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+import org.apache.hadoop.yarn.server.api.ContainerType;
import com.google.common.annotations.VisibleForTesting;
@@ -119,7 +120,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
public void initializeContainer(ContainerInitializationContext context) {
// intercept the event of the AM container being created and initialize the
// app level collector service
- if (isApplicationMaster(context)) {
+ if (context.getContainerType() == ContainerType.APPLICATION_MASTER) {
ApplicationId appId = context.getContainerId().
getApplicationAttemptId().getApplicationId();
addApplication(appId);
@@ -135,21 +136,13 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
public void stopContainer(ContainerTerminationContext context) {
// intercept the event of the AM container being stopped and remove the app
// level collector service
- if (isApplicationMaster(context)) {
+ if (context.getContainerType() == ContainerType.APPLICATION_MASTER) {
ApplicationId appId = context.getContainerId().
getApplicationAttemptId().getApplicationId();
removeApplication(appId);
}
}
- private boolean isApplicationMaster(ContainerContext context) {
- // TODO this is based on a (shaky) assumption that the container id (the
- // last field of the full container id) for an AM is always 1
- // we want to make this much more reliable
- ContainerId containerId = context.getContainerId();
- return containerId.getContainerId() == 1L;
- }
-
@VisibleForTesting
boolean hasApplication(ApplicationId appId) {
return collectorManager.containsTimelineCollector(appId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb57d79c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index b1a5b04..7cc612d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.junit.After;
@@ -94,6 +95,8 @@ public class TestPerNodeTimelineCollectorsAuxService {
ContainerTerminationContext context =
mock(ContainerTerminationContext.class);
when(context.getContainerId()).thenReturn(containerId);
+ when(context.getContainerType()).thenReturn(
+ ContainerType.APPLICATION_MASTER);
auxService.stopContainer(context);
// auxService should not have that app
assertFalse(auxService.hasApplication(appAttemptId.getApplicationId()));
@@ -138,6 +141,8 @@ public class TestPerNodeTimelineCollectorsAuxService {
ContainerInitializationContext context =
mock(ContainerInitializationContext.class);
when(context.getContainerId()).thenReturn(containerId);
+ when(context.getContainerType()).thenReturn(
+ ContainerType.APPLICATION_MASTER);
auxService.initializeContainer(context);
return auxService;
}
[45/50] [abbrv] hadoop git commit: YARN-3864. Implement support for
querying single app and all apps for a flow run (Varun Saxena via sjlee)
Posted by sj...@apache.org.
YARN-3864. Implement support for querying single app and all apps for a flow run (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cde783f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cde783f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cde783f7
Branch: refs/heads/YARN-2928-rebase
Commit: cde783f72c241550f2245e6bcf696392641d64a7
Parents: 529de2e
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Oct 5 13:14:11 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 17:05:05 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../reader/TimelineReaderWebServices.java | 206 +++++-
.../storage/ApplicationEntityReader.java | 64 +-
.../storage/FlowActivityEntityReader.java | 33 +-
.../storage/FlowRunEntityReader.java | 2 +-
.../storage/GenericEntityReader.java | 16 +-
.../storage/TimelineEntityReader.java | 21 +-
.../storage/TimelineEntityReaderFactory.java | 2 +-
.../storage/application/ApplicationRowKey.java | 34 +
.../TestTimelineReaderWebServicesFlowRun.java | 405 -----------
...stTimelineReaderWebServicesHBaseStorage.java | 673 +++++++++++++++++++
11 files changed, 1000 insertions(+), 459 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 066b243..969fcec 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -118,6 +118,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-4210. HBase reader throws NPE if Get returns no rows (Varun Saxena
via vrushali)
+ YARN-3864. Implement support for querying single app and all apps for a
+ flow run (Varun Saxena via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index a327099..610f74c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -286,7 +286,7 @@ public class TimelineReaderWebServices {
@QueryParam("eventfilters") String eventfilters,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
- (null == req.getQueryString() ? "" :
+ (req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = getUser(req);
LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
@@ -310,7 +310,7 @@ public class TimelineReaderWebServices {
parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime,
- "createdTime or modifiedTime start/end or limit or flowId");
+ "createdTime or modifiedTime start/end or limit or flowrunid");
}
long endTime = Time.monotonicNow();
if (entities == null) {
@@ -360,7 +360,7 @@ public class TimelineReaderWebServices {
@QueryParam("flowrunid") String flowRunId,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
- (null == req.getQueryString() ? "" :
+ (req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = getUser(req);
LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
@@ -420,7 +420,7 @@ public class TimelineReaderWebServices {
@QueryParam("userid") String userId,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
- (null == req.getQueryString() ? "" :
+ (req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = getUser(req);
LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
@@ -477,7 +477,7 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
- (null == req.getQueryString() ? "" :
+ (req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = getUser(req);
LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
@@ -502,4 +502,200 @@ public class TimelineReaderWebServices {
" (Took " + (endTime - startTime) + " ms.)");
return entities;
}
+
+ /**
+ * Return a single app for given app id. Cluster ID is not provided by
+ * client so default cluster ID has to be taken.
+ */
+ @GET
+ @Path("/app/{appid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public TimelineEntity getApp(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("appid") String appId,
+ @QueryParam("flowid") String flowId,
+ @QueryParam("flowrunid") String flowRunId,
+ @QueryParam("userid") String userId,
+ @QueryParam("fields") String fields) {
+ return getApp(req, res, null, appId, flowId, flowRunId, userId, fields);
+ }
+
+ /**
+ * Return a single app for given cluster id and app id.
+ */
+ @GET
+ @Path("/app/{clusterid}/{appid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public TimelineEntity getApp(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("clusterid") String clusterId,
+ @PathParam("appid") String appId,
+ @QueryParam("flowid") String flowId,
+ @QueryParam("flowrunid") String flowRunId,
+ @QueryParam("userid") String userId,
+ @QueryParam("fields") String fields) {
+ String url = req.getRequestURI() +
+ (req.getQueryString() == null ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI = getUser(req);
+ LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ TimelineEntity entity = null;
+ try {
+ entity = timelineReaderManager.getEntity(
+ parseUser(callerUGI, userId), parseStr(clusterId),
+ parseStr(flowId), parseLongStr(flowRunId), parseStr(appId),
+ TimelineEntityType.YARN_APPLICATION.toString(), null,
+ parseFieldsStr(fields, COMMA_DELIMITER));
+ } catch (Exception e) {
+ handleException(e, url, startTime, "flowrunid");
+ }
+ long endTime = Time.monotonicNow();
+ if (entity == null) {
+ LOG.info("Processed URL " + url + " but app not found" + " (Took " +
+ (endTime - startTime) + " ms.)");
+ throw new NotFoundException("App " + appId + " not found");
+ }
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
+ return entity;
+ }
+
+ /**
+ * Return a list of apps for given flow id and flow run id. Cluster ID is not
+ * provided by client so default cluster ID has to be taken. If number of
+ * matching apps are more than the limit, most recent apps till the limit is
+ * reached, will be returned.
+ */
+ @GET
+ @Path("/flowrunapps/{flowid}/{flowrunid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getFlowRunApps(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("flowid") String flowId,
+ @PathParam("flowrunid") String flowRunId,
+ @QueryParam("userid") String userId,
+ @QueryParam("limit") String limit,
+ @QueryParam("createdtimestart") String createdTimeStart,
+ @QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("modifiedtimestart") String modifiedTimeStart,
+ @QueryParam("modifiedtimeend") String modifiedTimeEnd,
+ @QueryParam("relatesto") String relatesTo,
+ @QueryParam("isrelatedto") String isRelatedTo,
+ @QueryParam("infofilters") String infofilters,
+ @QueryParam("conffilters") String conffilters,
+ @QueryParam("metricfilters") String metricfilters,
+ @QueryParam("eventfilters") String eventfilters,
+ @QueryParam("fields") String fields) {
+ return getEntities(req, res, null, null,
+ TimelineEntityType.YARN_APPLICATION.toString(), userId, flowId,
+ flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
+ modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
+ metricfilters, eventfilters, fields);
+ }
+
+ /**
+ * Return a list of apps for a given cluster id, flow id and flow run id. If
+ * number of matching apps are more than the limit, most recent apps till the
+ * limit is reached, will be returned.
+ */
+ @GET
+ @Path("/flowrunapps/{clusterid}/{flowid}/{flowrunid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getFlowRunApps(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("clusterid") String clusterId,
+ @PathParam("flowid") String flowId,
+ @PathParam("flowrunid") String flowRunId,
+ @QueryParam("userid") String userId,
+ @QueryParam("limit") String limit,
+ @QueryParam("createdtimestart") String createdTimeStart,
+ @QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("modifiedtimestart") String modifiedTimeStart,
+ @QueryParam("modifiedtimeend") String modifiedTimeEnd,
+ @QueryParam("relatesto") String relatesTo,
+ @QueryParam("isrelatedto") String isRelatedTo,
+ @QueryParam("infofilters") String infofilters,
+ @QueryParam("conffilters") String conffilters,
+ @QueryParam("metricfilters") String metricfilters,
+ @QueryParam("eventfilters") String eventfilters,
+ @QueryParam("fields") String fields) {
+ return getEntities(req, res, clusterId, null,
+ TimelineEntityType.YARN_APPLICATION.toString(), userId, flowId,
+ flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
+ modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
+ metricfilters, eventfilters, fields);
+ }
+
+ /**
+ * Return a list of apps for given flow id. Cluster ID is not provided by
+ * client so default cluster ID has to be taken. If number of matching apps
+ * are more than the limit, most recent apps till the limit is reached, will
+ * be returned.
+ */
+ @GET
+ @Path("/flowapps/{flowid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getFlowApps(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("flowid") String flowId,
+ @QueryParam("userid") String userId,
+ @QueryParam("limit") String limit,
+ @QueryParam("createdtimestart") String createdTimeStart,
+ @QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("modifiedtimestart") String modifiedTimeStart,
+ @QueryParam("modifiedtimeend") String modifiedTimeEnd,
+ @QueryParam("relatesto") String relatesTo,
+ @QueryParam("isrelatedto") String isRelatedTo,
+ @QueryParam("infofilters") String infofilters,
+ @QueryParam("conffilters") String conffilters,
+ @QueryParam("metricfilters") String metricfilters,
+ @QueryParam("eventfilters") String eventfilters,
+ @QueryParam("fields") String fields) {
+ return getEntities(req, res, null, null,
+ TimelineEntityType.YARN_APPLICATION.toString(), userId, flowId,
+ null, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
+ modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
+ metricfilters, eventfilters, fields);
+ }
+
+ /**
+ * Return a list of apps for a given cluster id and flow id. If number of
+ * matching apps are more than the limit, most recent apps till the limit is
+ * reached, will be returned.
+ */
+ @GET
+ @Path("/flowapps/{clusterid}/{flowid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getFlowApps(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("clusterid") String clusterId,
+ @PathParam("flowid") String flowId,
+ @QueryParam("userid") String userId,
+ @QueryParam("limit") String limit,
+ @QueryParam("createdtimestart") String createdTimeStart,
+ @QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("modifiedtimestart") String modifiedTimeStart,
+ @QueryParam("modifiedtimeend") String modifiedTimeEnd,
+ @QueryParam("relatesto") String relatesTo,
+ @QueryParam("isrelatedto") String isRelatedTo,
+ @QueryParam("infofilters") String infofilters,
+ @QueryParam("conffilters") String conffilters,
+ @QueryParam("metricfilters") String metricfilters,
+ @QueryParam("eventfilters") String eventfilters,
+ @QueryParam("fields") String fields) {
+ return getEntities(req, res, clusterId, null,
+ TimelineEntityType.YARN_APPLICATION.toString(), userId, flowId,
+ null, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
+ modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
+ metricfilters, eventfilters, fields);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
index d5b5d63..61954e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
-import java.util.Collections;
import java.util.EnumSet;
import java.util.Map;
import java.util.Set;
@@ -28,6 +27,8 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
@@ -38,6 +39,8 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.application.Applica
import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
+import com.google.common.base.Preconditions;
+
/**
* Timeline entity reader for application entities that are stored in the
* application table.
@@ -57,7 +60,7 @@ class ApplicationEntityReader extends GenericEntityReader {
super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, fieldsToRetrieve);
+ eventFilters, fieldsToRetrieve, true);
}
public ApplicationEntityReader(String userId, String clusterId,
@@ -86,10 +89,63 @@ class ApplicationEntityReader extends GenericEntityReader {
}
@Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(userId, "userId shouldn't be null");
+ Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
+ if (singleEntityRead) {
+ Preconditions.checkNotNull(appId, "appId shouldn't be null");
+ } else {
+ Preconditions.checkNotNull(flowId, "flowId shouldn't be null");
+ }
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ if (singleEntityRead) {
+ if (flowId == null || flowRunId == null) {
+ FlowContext context =
+ lookupFlowContext(clusterId, appId, hbaseConf, conn);
+ flowId = context.flowId;
+ flowRunId = context.flowRunId;
+ }
+ }
+ if (fieldsToRetrieve == null) {
+ fieldsToRetrieve = EnumSet.noneOf(Field.class);
+ }
+ if (!singleEntityRead) {
+ if (limit == null || limit < 0) {
+ limit = TimelineReader.DEFAULT_LIMIT;
+ }
+ if (createdTimeBegin == null) {
+ createdTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (createdTimeEnd == null) {
+ createdTimeEnd = DEFAULT_END_TIME;
+ }
+ if (modifiedTimeBegin == null) {
+ modifiedTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (modifiedTimeEnd == null) {
+ modifiedTimeEnd = DEFAULT_END_TIME;
+ }
+ }
+ }
+
+ @Override
protected ResultScanner getResults(Configuration hbaseConf,
Connection conn) throws IOException {
- throw new UnsupportedOperationException(
- "we don't support multiple apps query");
+ Scan scan = new Scan();
+ if (flowRunId != null) {
+ scan.setRowPrefixFilter(ApplicationRowKey.
+ getRowKeyPrefix(clusterId, userId, flowId, flowRunId));
+ } else {
+ scan.setRowPrefixFilter(ApplicationRowKey.
+ getRowKeyPrefix(clusterId, userId, flowId));
+ }
+ scan.setFilter(new PageFilter(limit));
+ return table.getResultScanner(hbaseConf, conn, scan);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
index e68ca17..70a0915 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
@@ -20,9 +20,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
import java.util.EnumSet;
import java.util.Map;
-import java.util.NavigableSet;
import java.util.Set;
-import java.util.TreeSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
@@ -60,7 +58,7 @@ class FlowActivityEntityReader extends TimelineEntityReader {
super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, fieldsToRetrieve);
+ eventFilters, fieldsToRetrieve, true);
}
public FlowActivityEntityReader(String userId, String clusterId,
@@ -78,35 +76,6 @@ class FlowActivityEntityReader extends TimelineEntityReader {
return FLOW_ACTIVITY_TABLE;
}
- /**
- * Since this is strictly sorted by the row key, it is sufficient to collect
- * the first results as specified by the limit.
- */
- @Override
- public Set<TimelineEntity> readEntities(Configuration hbaseConf,
- Connection conn) throws IOException {
- validateParams();
- augmentParams(hbaseConf, conn);
-
- NavigableSet<TimelineEntity> entities = new TreeSet<>();
- ResultScanner results = getResults(hbaseConf, conn);
- try {
- for (Result result : results) {
- TimelineEntity entity = parseEntity(result);
- if (entity == null) {
- continue;
- }
- entities.add(entity);
- if (entities.size() == limit) {
- break;
- }
- }
- return entities;
- } finally {
- results.close();
- }
- }
-
@Override
protected void validateParams() {
Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
index b5d7ae5..90ce28f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
@@ -56,7 +56,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, fieldsToRetrieve);
+ eventFilters, fieldsToRetrieve, false);
}
public FlowRunEntityReader(String userId, String clusterId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
index 396a02b..42079d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
@@ -61,8 +61,8 @@ class GenericEntityReader extends TimelineEntityReader {
private static final EntityTable ENTITY_TABLE = new EntityTable();
private static final Log LOG = LogFactory.getLog(GenericEntityReader.class);
- private static final long DEFAULT_BEGIN_TIME = 0L;
- private static final long DEFAULT_END_TIME = Long.MAX_VALUE;
+ protected static final long DEFAULT_BEGIN_TIME = 0L;
+ protected static final long DEFAULT_END_TIME = Long.MAX_VALUE;
/**
* Used to look up the flow context.
@@ -76,11 +76,11 @@ class GenericEntityReader extends TimelineEntityReader {
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
- EnumSet<Field> fieldsToRetrieve) {
+ EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) {
super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, fieldsToRetrieve);
+ eventFilters, fieldsToRetrieve, sortedKeys);
}
public GenericEntityReader(String userId, String clusterId,
@@ -97,7 +97,7 @@ class GenericEntityReader extends TimelineEntityReader {
return ENTITY_TABLE;
}
- private FlowContext lookupFlowContext(String clusterId, String appId,
+ protected FlowContext lookupFlowContext(String clusterId, String appId,
Configuration hbaseConf, Connection conn) throws IOException {
byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
Get get = new Get(rowKey);
@@ -113,9 +113,9 @@ class GenericEntityReader extends TimelineEntityReader {
}
}
- private static class FlowContext {
- private final String flowId;
- private final Long flowRunId;
+ protected static class FlowContext {
+ protected final String flowId;
+ protected final Long flowRunId;
public FlowContext(String flowId, Long flowRunId) {
this.flowId = flowId;
this.flowRunId = flowRunId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
index 93be2db..d4a659c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
@@ -74,6 +74,14 @@ abstract class TimelineEntityReader {
protected BaseTable<?> table;
/**
+ * Specifies whether keys for this table are sorted in a manner where entities
+ * can be retrieved by created time. If true, it will be sufficient to collect
+ * the first results as specified by the limit. Otherwise all matched entities
+ * will be fetched and then limit applied.
+ */
+ private boolean sortedKeys = false;
+
+ /**
* Instantiates a reader for multiple-entity reads.
*/
protected TimelineEntityReader(String userId, String clusterId,
@@ -83,8 +91,9 @@ abstract class TimelineEntityReader {
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
- EnumSet<Field> fieldsToRetrieve) {
+ EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) {
this.singleEntityRead = false;
+ this.sortedKeys = sortedKeys;
this.userId = userId;
this.clusterId = clusterId;
this.flowId = flowId;
@@ -162,8 +171,14 @@ abstract class TimelineEntityReader {
continue;
}
entities.add(entity);
- if (entities.size() > limit) {
- entities.pollLast();
+ if (!sortedKeys) {
+ if (entities.size() > limit) {
+ entities.pollLast();
+ }
+ } else {
+ if (entities.size() == limit) {
+ break;
+ }
}
}
return entities;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
index 4fdef40..f5341c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
@@ -91,7 +91,7 @@ class TimelineEntityReaderFactory {
appId, entityType, limit, createdTimeBegin, createdTimeEnd,
modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
infoFilters, configFilters, metricFilters, eventFilters,
- fieldsToRetrieve);
+ fieldsToRetrieve, false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
index e3b5a87..10e3c2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
@@ -61,6 +61,40 @@ public class ApplicationRowKey {
}
/**
+ * Constructs a row key prefix for the application table as follows:
+ * {@code clusterId!userName!flowId!}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @return byte array with the row key prefix
+ */
+ public static byte[] getRowKeyPrefix(String clusterId, String userId,
+ String flowId) {
+ byte[] first = Bytes.toBytes(
+ Separator.QUALIFIERS.joinEncoded(clusterId, userId, flowId));
+ return Separator.QUALIFIERS.join(first, new byte[0]);
+ }
+
+ /**
+ * Constructs a row key prefix for the application table as follows:
+ * {@code clusterId!userName!flowId!flowRunId!}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @return byte array with the row key prefix
+ */
+ public static byte[] getRowKeyPrefix(String clusterId, String userId,
+ String flowId, Long flowRunId) {
+ byte[] first = Bytes.toBytes(
+ Separator.QUALIFIERS.joinEncoded(clusterId, userId, flowId));
+ byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
+ return Separator.QUALIFIERS.join(first, second, new byte[0]);
+ }
+
+ /**
* Constructs a row key for the application table as follows:
* {@code clusterId!userName!flowId!flowRunId!AppId}
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java
deleted file mode 100644
index e359f78..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java
+++ /dev/null
@@ -1,405 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.reader;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import javax.ws.rs.core.MediaType;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
-import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.ClientResponse.Status;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-
-public class TestTimelineReaderWebServicesFlowRun {
- private int serverPort;
- private TimelineReaderServer server;
- private static HBaseTestingUtility util;
- private static long ts = System.currentTimeMillis();
-
- @BeforeClass
- public static void setup() throws Exception {
- util = new HBaseTestingUtility();
- Configuration conf = util.getConfiguration();
- conf.setInt("hfile.format.version", 3);
- util.startMiniCluster();
- TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
- loadData();
- }
-
- private static void loadData() throws Exception {
- String cluster = "cluster1";
- String user = "user1";
- String flow = "flow_name";
- String flowVersion = "CF7022C10F1354";
- Long runid = 1002345678919L;
- Long runid1 = 1002345678920L;
-
- TimelineEntities te = new TimelineEntities();
- TimelineEntity entity = new TimelineEntity();
- String id = "flowRunMetrics_test";
- String type = TimelineEntityType.YARN_APPLICATION.toString();
- entity.setId(id);
- entity.setType(type);
- Long cTime = 1425016501000L;
- entity.setCreatedTime(cTime);
-
- // add metrics
- Set<TimelineMetric> metrics = new HashSet<>();
- TimelineMetric m1 = new TimelineMetric();
- m1.setId("MAP_SLOT_MILLIS");
- Map<Long, Number> metricValues = new HashMap<Long, Number>();
- metricValues.put(ts - 100000, 2);
- metricValues.put(ts - 80000, 40);
- m1.setType(Type.TIME_SERIES);
- m1.setValues(metricValues);
- metrics.add(m1);
-
- m1 = new TimelineMetric();
- m1.setId("HDFS_BYTES_READ");
- metricValues = new HashMap<Long, Number>();
- metricValues.put(ts - 100000, 31);
- metricValues.put(ts - 80000, 57);
- m1.setType(Type.TIME_SERIES);
- m1.setValues(metricValues);
- metrics.add(m1);
- entity.addMetrics(metrics);
-
- TimelineEvent event = new TimelineEvent();
- event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- Long expTs = 1436512802000L;
- event.setTimestamp(expTs);
- String expKey = "foo_event";
- Object expVal = "test";
- event.addInfo(expKey, expVal);
- entity.addEvent(event);
- te.addEntity(entity);
-
- // write another application with same metric to this flow
- TimelineEntities te1 = new TimelineEntities();
- TimelineEntity entity1 = new TimelineEntity();
- id = "flowRunMetrics_test";
- type = TimelineEntityType.YARN_APPLICATION.toString();
- entity1.setId(id);
- entity1.setType(type);
- cTime = 1425016501000L;
- entity1.setCreatedTime(cTime);
- // add metrics
- metrics.clear();
- TimelineMetric m2 = new TimelineMetric();
- m2.setId("MAP_SLOT_MILLIS");
- metricValues = new HashMap<Long, Number>();
- metricValues.put(ts - 100000, 5L);
- metricValues.put(ts - 80000, 101L);
- m2.setType(Type.TIME_SERIES);
- m2.setValues(metricValues);
- metrics.add(m2);
- entity1.addMetrics(metrics);
- te1.addEntity(entity1);
-
- String flow2 = "flow_name2";
- String flowVersion2 = "CF7022C10F1454";
- Long runid2 = 2102356789046L;
- TimelineEntities te3 = new TimelineEntities();
- TimelineEntity entity3 = new TimelineEntity();
- id = "flowRunMetrics_test1";
- entity3.setId(id);
- entity3.setType(type);
- cTime = 1425016501030L;
- entity3.setCreatedTime(cTime);
- TimelineEvent event2 = new TimelineEvent();
- event2.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- event2.setTimestamp(1436512802030L);
- event2.addInfo("foo_event", "test");
- entity3.addEvent(event2);
- te3.addEntity(entity3);
-
- HBaseTimelineWriterImpl hbi = null;
- Configuration c1 = util.getConfiguration();
- try {
- hbi = new HBaseTimelineWriterImpl(c1);
- hbi.init(c1);
- String appName = "application_11111111111111_1111";
- hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
- appName = "application_11111111111111_2222";
- hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
- hbi.write(cluster, user, flow, flowVersion, runid1, appName, te);
- appName = "application_11111111111111_2223";
- hbi.write(cluster, user, flow2, flowVersion2, runid2, appName, te3);
- hbi.flush();
- } finally {
- hbi.close();
- }
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- util.shutdownMiniCluster();
- }
-
- @Before
- public void init() throws Exception {
- try {
- Configuration config = util.getConfiguration();
- config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
- "localhost:0");
- config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
- config.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
- "org.apache.hadoop.yarn.server.timelineservice.storage." +
- "HBaseTimelineReaderImpl");
- config.setInt("hfile.format.version", 3);
- server = new TimelineReaderServer();
- server.init(config);
- server.start();
- serverPort = server.getWebServerPort();
- } catch (Exception e) {
- Assert.fail("Web server failed to start");
- }
- }
-
- private static Client createClient() {
- ClientConfig cfg = new DefaultClientConfig();
- cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
- return new Client(new URLConnectionClientHandler(
- new DummyURLConnectionFactory()), cfg);
- }
-
- private static ClientResponse getResponse(Client client, URI uri)
- throws Exception {
- ClientResponse resp =
- client.resource(uri).accept(MediaType.APPLICATION_JSON)
- .type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
- if (resp == null ||
- resp.getClientResponseStatus() != ClientResponse.Status.OK) {
- String msg = new String();
- if (resp != null) {
- msg = resp.getClientResponseStatus().toString();
- }
- throw new IOException("Incorrect response from timeline reader. " +
- "Status=" + msg);
- }
- return resp;
- }
-
- private static class DummyURLConnectionFactory
- implements HttpURLConnectionFactory {
-
- @Override
- public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
- try {
- return (HttpURLConnection)url.openConnection();
- } catch (UndeclaredThrowableException e) {
- throw new IOException(e.getCause());
- }
- }
- }
-
- private static TimelineMetric newMetric(String id, long ts, Number value) {
- TimelineMetric metric = new TimelineMetric();
- metric.setId(id);
- metric.addValue(ts, value);
- return metric;
- }
-
- private static boolean verifyMetricValues(Map<Long, Number> m1,
- Map<Long, Number> m2) {
- for (Map.Entry<Long, Number> entry : m1.entrySet()) {
- if (!m2.containsKey(entry.getKey())) {
- return false;
- }
- if (m2.get(entry.getKey()).equals(entry.getValue())) {
- return false;
- }
- }
- return true;
- }
-
- private static boolean verifyMetrics(
- TimelineMetric m, TimelineMetric... metrics) {
- for (TimelineMetric metric : metrics) {
- if (!metric.equals(m)) {
- continue;
- }
- if (!verifyMetricValues(metric.getValues(), m.getValues())) {
- continue;
- }
- return true;
- }
- return false;
- }
-
- private static void verifyHttpResponse(Client client, URI uri,
- Status status) {
- ClientResponse resp =
- client.resource(uri).accept(MediaType.APPLICATION_JSON)
- .type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
- assertNotNull(resp);
- assertTrue("Response from server should have been " + status,
- resp.getClientResponseStatus().equals(status));
- }
-
- @Test
- public void testGetFlowRun() throws Exception {
- Client client = createClient();
- try {
- URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowrun/cluster1/flow_name/1002345678919?userid=user1");
- ClientResponse resp = getResponse(client, uri);
- FlowRunEntity entity = resp.getEntity(FlowRunEntity.class);
- assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
- assertNotNull(entity);
- assertEquals("user1@flow_name/1002345678919", entity.getId());
- assertEquals(2, entity.getMetrics().size());
- TimelineMetric m1 = newMetric("HDFS_BYTES_READ", ts - 80000, 57L);
- TimelineMetric m2 = newMetric("MAP_SLOT_MILLIS", ts - 80000, 141L);
- for (TimelineMetric metric : entity.getMetrics()) {
- assertTrue(verifyMetrics(metric, m1, m2));
- }
-
- // Query without specifying cluster ID.
- uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowrun/flow_name/1002345678919?userid=user1");
- resp = getResponse(client, uri);
- entity = resp.getEntity(FlowRunEntity.class);
- assertNotNull(entity);
- assertEquals("user1@flow_name/1002345678919", entity.getId());
- assertEquals(2, entity.getMetrics().size());
- m1 = newMetric("HDFS_BYTES_READ", ts - 80000, 57L);
- m2 = newMetric("MAP_SLOT_MILLIS", ts - 80000, 141L);
- for (TimelineMetric metric : entity.getMetrics()) {
- assertTrue(verifyMetrics(metric, m1, m2));
- }
- } finally {
- client.destroy();
- }
- }
-
- @Test
- public void testGetFlows() throws Exception {
- Client client = createClient();
- try {
- URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1");
- ClientResponse resp = getResponse(client, uri);
- Set<FlowActivityEntity> entities =
- resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
- assertNotNull(entities);
- assertEquals(2, entities.size());
- for (FlowActivityEntity entity : entities) {
- assertTrue((entity.getId().endsWith("@flow_name") &&
- entity.getFlowRuns().size() == 2) ||
- (entity.getId().endsWith("@flow_name2") &&
- entity.getFlowRuns().size() == 1));
- }
-
- // Query without specifying cluster ID.
- uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/");
- resp = getResponse(client, uri);
- entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
- assertNotNull(entities);
- assertEquals(2, entities.size());
-
- uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1?limit=1");
- resp = getResponse(client, uri);
- entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
- assertNotNull(entities);
- assertEquals(1, entities.size());
- } finally {
- client.destroy();
- }
- }
-
- @Test
- public void testGetFlowRunNotPresent() throws Exception {
- Client client = createClient();
- try {
- URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowrun/cluster1/flow_name/1002345678929?userid=user1");
- verifyHttpResponse(client, uri, Status.NOT_FOUND);
- } finally {
- client.destroy();
- }
- }
-
- @Test
- public void testGetFlowsNotPresent() throws Exception {
- Client client = createClient();
- try {
- URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster2");
- ClientResponse resp = getResponse(client, uri);
- Set<FlowActivityEntity> entities =
- resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
- assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
- assertNotNull(entities);
- assertEquals(0, entities.size());
- } finally {
- client.destroy();
- }
- }
-
- @After
- public void stop() throws Exception {
- if (server != null) {
- server.stop();
- server = null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde783f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
new file mode 100644
index 0000000..a89d2fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -0,0 +1,673 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.core.MediaType;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+
+public class TestTimelineReaderWebServicesHBaseStorage {
+ private int serverPort;
+ private TimelineReaderServer server;
+ private static HBaseTestingUtility util;
+ private static long ts = System.currentTimeMillis();
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ util = new HBaseTestingUtility();
+ Configuration conf = util.getConfiguration();
+ conf.setInt("hfile.format.version", 3);
+ util.startMiniCluster();
+ TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
+ loadData();
+ }
+
+ private static void loadData() throws Exception {
+ String cluster = "cluster1";
+ String user = "user1";
+ String flow = "flow_name";
+ String flowVersion = "CF7022C10F1354";
+ Long runid = 1002345678919L;
+ Long runid1 = 1002345678920L;
+
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entity = new TimelineEntity();
+ String id = "application_1111111111_1111";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ Long cTime = 1425016501000L;
+ entity.setCreatedTime(cTime);
+ entity.addConfig("cfg2", "value1");
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ metricValues.put(ts - 100000, 2);
+ metricValues.put(ts - 80000, 40);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+
+ m1 = new TimelineMetric();
+ m1.setId("HDFS_BYTES_READ");
+ metricValues = new HashMap<Long, Number>();
+ metricValues.put(ts - 100000, 31);
+ metricValues.put(ts - 80000, 57);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ Long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+ String expKey = "foo_event";
+ Object expVal = "test";
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
+ TimelineEvent event11 = new TimelineEvent();
+ event11.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ expTs = 1436512802010L;
+ event11.setTimestamp(expTs);
+ entity.addEvent(event11);
+
+ te.addEntity(entity);
+
+ // write another application with same metric to this flow
+ TimelineEntities te1 = new TimelineEntities();
+ TimelineEntity entity1 = new TimelineEntity();
+ id = "application_1111111111_2222";
+ type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity1.setId(id);
+ entity1.setType(type);
+ cTime = 1425016501000L;
+ entity1.setCreatedTime(cTime);
+ entity1.addConfig("cfg1", "value1");
+ // add metrics
+ metrics.clear();
+ TimelineMetric m2 = new TimelineMetric();
+ m2.setId("MAP_SLOT_MILLIS");
+ metricValues = new HashMap<Long, Number>();
+ metricValues.put(ts - 100000, 5L);
+ metricValues.put(ts - 80000, 101L);
+ m2.setType(Type.TIME_SERIES);
+ m2.setValues(metricValues);
+ metrics.add(m2);
+ entity1.addMetrics(metrics);
+ TimelineEvent event1 = new TimelineEvent();
+ event1.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ event1.setTimestamp(expTs);
+ event1.addInfo(expKey, expVal);
+ entity1.addEvent(event1);
+ te1.addEntity(entity1);
+
+ String flow2 = "flow_name2";
+ String flowVersion2 = "CF7022C10F1454";
+ Long runid2 = 2102356789046L;
+ TimelineEntities te3 = new TimelineEntities();
+ TimelineEntity entity3 = new TimelineEntity();
+ id = "application_11111111111111_2223";
+ entity3.setId(id);
+ entity3.setType(type);
+ cTime = 1425016501030L;
+ entity3.setCreatedTime(cTime);
+ TimelineEvent event2 = new TimelineEvent();
+ event2.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ event2.setTimestamp(1436512802030L);
+ event2.addInfo("foo_event", "test");
+ entity3.addEvent(event2);
+ te3.addEntity(entity3);
+
+ TimelineEntities te4 = new TimelineEntities();
+ TimelineEntity entity4 = new TimelineEntity();
+ id = "application_1111111111_2224";
+ entity4.setId(id);
+ entity4.setType(type);
+ cTime = 1425016501034L;
+ entity4.setCreatedTime(cTime);
+ TimelineEvent event4 = new TimelineEvent();
+ event4.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ event4.setTimestamp(1436512802037L);
+ event4.addInfo("foo_event", "test");
+ entity4.addEvent(event4);
+ te4.addEntity(entity4);
+
+ HBaseTimelineWriterImpl hbi = null;
+ Configuration c1 = util.getConfiguration();
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.write(cluster, user, flow, flowVersion, runid, entity.getId(), te);
+ hbi.write(cluster, user, flow, flowVersion, runid, entity1.getId(), te1);
+ hbi.write(cluster, user, flow, flowVersion, runid1, entity4.getId(), te4);
+ hbi.write(cluster, user, flow2,
+ flowVersion2, runid2, entity3.getId(), te3);
+ hbi.flush();
+ } finally {
+ hbi.close();
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ util.shutdownMiniCluster();
+ }
+
+ @Before
+ public void init() throws Exception {
+ try {
+ Configuration config = util.getConfiguration();
+ config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ "localhost:0");
+ config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
+ config.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
+ "org.apache.hadoop.yarn.server.timelineservice.storage." +
+ "HBaseTimelineReaderImpl");
+ config.setInt("hfile.format.version", 3);
+ server = new TimelineReaderServer();
+ server.init(config);
+ server.start();
+ serverPort = server.getWebServerPort();
+ } catch (Exception e) {
+ Assert.fail("Web server failed to start");
+ }
+ }
+
+ private static Client createClient() {
+ ClientConfig cfg = new DefaultClientConfig();
+ cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
+ return new Client(new URLConnectionClientHandler(
+ new DummyURLConnectionFactory()), cfg);
+ }
+
+ private static ClientResponse getResponse(Client client, URI uri)
+ throws Exception {
+ ClientResponse resp =
+ client.resource(uri).accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ if (resp == null ||
+ resp.getClientResponseStatus() != ClientResponse.Status.OK) {
+ String msg = new String();
+ if (resp != null) {
+ msg = resp.getClientResponseStatus().toString();
+ }
+ throw new IOException("Incorrect response from timeline reader. " +
+ "Status=" + msg);
+ }
+ return resp;
+ }
+
+ private static class DummyURLConnectionFactory
+ implements HttpURLConnectionFactory {
+
+ @Override
+ public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
+ try {
+ return (HttpURLConnection)url.openConnection();
+ } catch (UndeclaredThrowableException e) {
+ throw new IOException(e.getCause());
+ }
+ }
+ }
+
+ private static TimelineEntity newEntity(String type, String id) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setIdentifier(new TimelineEntity.Identifier(type, id));
+ return entity;
+ }
+
+ private static TimelineMetric newMetric(TimelineMetric.Type type,
+ String id, long ts, Number value) {
+ TimelineMetric metric = new TimelineMetric(type);
+ metric.setId(id);
+ metric.addValue(ts, value);
+ return metric;
+ }
+
+ private static boolean verifyMetricValues(Map<Long, Number> m1,
+ Map<Long, Number> m2) {
+ for (Map.Entry<Long, Number> entry : m1.entrySet()) {
+ if (!m2.containsKey(entry.getKey())) {
+ return false;
+ }
+ if (m2.get(entry.getKey()).equals(entry.getValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean verifyMetrics(
+ TimelineMetric m, TimelineMetric... metrics) {
+ for (TimelineMetric metric : metrics) {
+ if (!metric.equals(m)) {
+ continue;
+ }
+ if (!verifyMetricValues(metric.getValues(), m.getValues())) {
+ continue;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private static void verifyHttpResponse(Client client, URI uri,
+ Status status) {
+ ClientResponse resp =
+ client.resource(uri).accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertNotNull(resp);
+ assertTrue("Response from server should have been " + status,
+ resp.getClientResponseStatus().equals(status));
+ }
+
+ @Test
+ public void testGetFlowRun() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowrun/cluster1/flow_name/1002345678919?userid=user1");
+ ClientResponse resp = getResponse(client, uri);
+ FlowRunEntity entity = resp.getEntity(FlowRunEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("user1@flow_name/1002345678919", entity.getId());
+ assertEquals(2, entity.getMetrics().size());
+ TimelineMetric m1 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
+ "HDFS_BYTES_READ", ts - 80000, 57L);
+ TimelineMetric m2 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
+ "MAP_SLOT_MILLIS", ts - 80000, 141L);
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(verifyMetrics(metric, m1, m2));
+ }
+
+ // Query without specifying cluster ID.
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowrun/flow_name/1002345678919?userid=user1");
+ resp = getResponse(client, uri);
+ entity = resp.getEntity(FlowRunEntity.class);
+ assertNotNull(entity);
+ assertEquals("user1@flow_name/1002345678919", entity.getId());
+ assertEquals(2, entity.getMetrics().size());
+ m1 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
+ "HDFS_BYTES_READ", ts - 80000, 57L);
+ m2 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
+ "MAP_SLOT_MILLIS", ts - 80000, 141L);
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(verifyMetrics(metric, m1, m2));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetFlows() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flows/cluster1");
+ ClientResponse resp = getResponse(client, uri);
+ Set<FlowActivityEntity> entities =
+ resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (FlowActivityEntity entity : entities) {
+ assertTrue((entity.getId().endsWith("@flow_name") &&
+ entity.getFlowRuns().size() == 2) ||
+ (entity.getId().endsWith("@flow_name2") &&
+ entity.getFlowRuns().size() == 1));
+ }
+
+ // Query without specifying cluster ID.
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flows/");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flows/cluster1?limit=1");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetApp() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/app/cluster1/application_1111111111_1111?" +
+ "userid=user1&fields=ALL&flowid=flow_name&flowrunid=1002345678919");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ assertEquals("application_1111111111_1111", entity.getId());
+ assertEquals(2, entity.getMetrics().size());
+ TimelineMetric m1 = newMetric(TimelineMetric.Type.TIME_SERIES,
+ "HDFS_BYTES_READ", ts - 100000, 31L);
+ m1.addValue(ts - 80000, 57L);
+ TimelineMetric m2 = newMetric(TimelineMetric.Type.TIME_SERIES,
+ "MAP_SLOT_MILLIS", ts - 100000, 2L);
+ m2.addValue(ts - 80000, 40L);
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(verifyMetrics(metric, m1, m2));
+ }
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/app/application_1111111111_2222?userid=user1" +
+ "&fields=metrics&flowid=flow_name&flowrunid=1002345678919");
+ resp = getResponse(client, uri);
+ entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ assertEquals("application_1111111111_2222", entity.getId());
+ assertEquals(1, entity.getMetrics().size());
+ TimelineMetric m3 = newMetric(TimelineMetric.Type.TIME_SERIES,
+ "MAP_SLOT_MILLIS", ts - 100000, 5L);
+ m2.addValue(ts - 80000, 101L);
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(verifyMetrics(metric, m3));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetAppWithoutFlowInfo() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/app/cluster1/application_1111111111_1111?" +
+ "userid=user1&fields=ALL");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ assertEquals("application_1111111111_1111", entity.getId());
+ assertEquals(2, entity.getMetrics().size());
+ TimelineMetric m1 = newMetric(TimelineMetric.Type.TIME_SERIES,
+ "HDFS_BYTES_READ", ts - 100000, 31L);
+ m1.addValue(ts - 80000, 57L);
+ TimelineMetric m2 = newMetric(TimelineMetric.Type.TIME_SERIES,
+ "MAP_SLOT_MILLIS", ts - 100000, 2L);
+ m2.addValue(ts - 80000, 40L);
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(verifyMetrics(metric, m1, m2));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetFlowRunApps() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowrunapps/cluster1/flow_name/1002345678919?" +
+ "userid=user1&fields=ALL");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue("Unexpected app in result",
+ (entity.getId().equals("application_1111111111_1111") &&
+ entity.getMetrics().size() == 2) ||
+ (entity.getId().equals("application_1111111111_2222") &&
+ entity.getMetrics().size() == 1));
+ }
+
+ // Query without specifying cluster ID.
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowrunapps/flow_name/1002345678919?userid=user1");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowrunapps/flow_name/1002345678919?userid=user1&limit=1");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetFlowApps() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowapps/cluster1/flow_name?userid=user1&fields=ALL");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(3, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue("Unexpected app in result",
+ (entity.getId().equals("application_1111111111_1111") &&
+ entity.getMetrics().size() == 2) ||
+ (entity.getId().equals("application_1111111111_2222") &&
+ entity.getMetrics().size() == 1) ||
+ (entity.getId().equals("application_1111111111_2224") &&
+ entity.getMetrics().size() == 0));
+ }
+
+ // Query without specifying cluster ID.
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowapps/flow_name?userid=user1");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(3, entities.size());
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowapps/flow_name?userid=user1&limit=1");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetFlowAppsFilters() throws Exception {
+ Client client = createClient();
+ try {
+ String entityType = TimelineEntityType.YARN_APPLICATION.toString();
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowapps/cluster1/flow_name?userid=user1&eventfilters=" +
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Unexpected app in result", entities.contains(
+ newEntity(entityType, "application_1111111111_1111")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowapps/cluster1/flow_name?userid=user1&metricfilters=" +
+ "HDFS_BYTES_READ");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Unexpected app in result", entities.contains(
+ newEntity(entityType, "application_1111111111_1111")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowapps/cluster1/flow_name?userid=user1&conffilters=" +
+ "cfg1:value1");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Unexpected app in result", entities.contains(
+ newEntity(entityType, "application_1111111111_2222")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetFlowRunNotPresent() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowrun/cluster1/flow_name/1002345678929?userid=user1");
+ verifyHttpResponse(client, uri, Status.NOT_FOUND);
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetFlowsNotPresent() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flows/cluster2");
+ ClientResponse resp = getResponse(client, uri);
+ Set<FlowActivityEntity> entities =
+ resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetAppNotPresent() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/app/cluster1/flow_name/1002345678919/" +
+ "application_1111111111_1378?userid=user1");
+ verifyHttpResponse(client, uri, Status.NOT_FOUND);
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetFlowRunAppsNotPresent() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowrunapps/cluster2/flow_name/1002345678919");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetFlowAppsNotPresent() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowapps/cluster2/flow_name55");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @After
+ public void stop() throws Exception {
+ if (server != null) {
+ server.stop();
+ server = null;
+ }
+ }
+}
[13/50] [abbrv] hadoop git commit: YARN-3044. Made RM write app,
attempt and optional container lifecycle events to timeline service
v2. Contributed by Naganarasimha G R.
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
new file mode 100644
index 0000000..9830a80
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -0,0 +1,374 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.AbstractTimelineServicePublisher.MultiThreadedDispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestSystemMetricsPublisherForV2 {
+
+ /**
+ * is the folder where the FileSystemTimelineWriterImpl writes the entities
+ */
+ protected static File testRootDir = new File("target",
+ TestSystemMetricsPublisherForV2.class.getName() + "-localDir")
+ .getAbsoluteFile();
+
+ private static SystemMetricsPublisher metricsPublisher;
+ private static DrainDispatcher dispatcher = new DrainDispatcher();
+ private static final String DEFAULT_FLOW_VERSION = "1";
+ private static final long DEFAULT_FLOW_RUN = 1;
+
+ private static ConcurrentMap<ApplicationId, RMApp> rmAppsMapInContext;
+
+ private static RMTimelineCollectorManager rmTimelineCollectorManager;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ if (testRootDir.exists()) {
+ //cleanup before hand
+ FileContext.getLocalFSFileContext().delete(
+ new Path(testRootDir.getAbsolutePath()), true);
+ }
+
+ RMContext rmContext = mock(RMContext.class);
+ rmAppsMapInContext = new ConcurrentHashMap<ApplicationId, RMApp>();
+ when(rmContext.getRMApps()).thenReturn(rmAppsMapInContext);
+ rmTimelineCollectorManager = new RMTimelineCollectorManager(rmContext);
+ when(rmContext.getRMTimelineCollectorManager()).thenReturn(
+ rmTimelineCollectorManager);
+
+ Configuration conf = getTimelineV2Conf();
+ rmTimelineCollectorManager.init(conf);
+ rmTimelineCollectorManager.start();
+
+ metricsPublisher = new SystemMetricsPublisher(rmContext) {
+ @Override
+ Dispatcher createDispatcher(
+ TimelineServicePublisher timelineServicePublisher) {
+ return dispatcher;
+ }
+ };
+ metricsPublisher.init(conf);
+ metricsPublisher.start();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ if (testRootDir.exists()) {
+ FileContext.getLocalFSFileContext().delete(
+ new Path(testRootDir.getAbsolutePath()), true);
+ }
+ if (rmTimelineCollectorManager != null) {
+ rmTimelineCollectorManager.stop();
+ }
+ if (metricsPublisher != null) {
+ metricsPublisher.stop();
+ }
+ }
+
+ private static Configuration getTimelineV2Conf() {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+ conf.setInt(
+ YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, 2);
+ conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
+ true);
+ try {
+ conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ testRootDir.getCanonicalPath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert
+ .fail("Exception while setting the TIMELINE_SERVICE_STORAGE_DIR_ROOT ");
+ }
+ return conf;
+ }
+
+ @Test
+ public void testSystemMetricPublisherInitialization() {
+ @SuppressWarnings("resource")
+ SystemMetricsPublisher metricsPublisher =
+ new SystemMetricsPublisher(mock(RMContext.class));
+ try {
+ Configuration conf = getTimelineV2Conf();
+ conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
+ false);
+ metricsPublisher.init(conf);
+ assertFalse(
+ "Default configuration should not publish container Metrics from RM",
+ metricsPublisher.isPublishContainerMetrics());
+
+ metricsPublisher.stop();
+
+ metricsPublisher = new SystemMetricsPublisher(mock(RMContext.class));
+ conf = getTimelineV2Conf();
+ metricsPublisher.init(conf);
+ assertTrue("Expected to publish container Metrics from RM",
+ metricsPublisher.isPublishContainerMetrics());
+ assertTrue(
+ "MultiThreadedDispatcher expected when container Metrics is not published",
+ metricsPublisher.getDispatcher() instanceof MultiThreadedDispatcher);
+ } finally {
+ metricsPublisher.stop();
+ }
+ }
+
+ @Test(timeout = 1000000)
+ public void testPublishApplicationMetrics() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ RMApp app = createAppAndRegister(appId);
+
+ metricsPublisher.appCreated(app, app.getStartTime());
+ metricsPublisher.appACLsUpdated(app, "user1,user2", 4L);
+ metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
+ dispatcher.await();
+
+ String outputDirApp =
+ getTimelineEntityDir(app) + "/" + TimelineEntityType.YARN_APPLICATION
+ + "/";
+
+ File entityFolder = new File(outputDirApp);
+ Assert.assertTrue(entityFolder.isDirectory());
+
+ // file name is <entityId>.thist
+ String timelineServiceFileName =
+ appId.toString()
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ File appFile = new File(outputDirApp, timelineServiceFileName);
+ Assert.assertTrue(appFile.exists());
+ Assert.assertEquals("Expected 3 events to be published", 3,
+ getNumOfNonEmptyLines(appFile));
+ }
+
+ @Test(timeout = 10000)
+ public void testPublishAppAttemptMetrics() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ RMApp app = rmAppsMapInContext.get(appId);
+ if (app == null) {
+ app = createAppAndRegister(appId);
+ }
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId);
+ metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
+ when(app.getFinalApplicationStatus()).thenReturn(
+ FinalApplicationStatus.UNDEFINED);
+ metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED,
+ app, Integer.MAX_VALUE + 2L);
+
+ dispatcher.await();
+
+ String outputDirApp =
+ getTimelineEntityDir(app) + "/"
+ + TimelineEntityType.YARN_APPLICATION_ATTEMPT + "/";
+
+ File entityFolder = new File(outputDirApp);
+ Assert.assertTrue(entityFolder.isDirectory());
+
+ // file name is <entityId>.thist
+ String timelineServiceFileName =
+ appAttemptId.toString()
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ File appFile = new File(outputDirApp, timelineServiceFileName);
+ Assert.assertTrue(appFile.exists());
+ Assert.assertEquals("Expected 2 events to be published", 2,
+ getNumOfNonEmptyLines(appFile));
+ }
+
+ @Test(timeout = 10000)
+ public void testPublishContainerMetrics() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ RMApp app = rmAppsMapInContext.get(appId);
+ if (app == null) {
+ app = createAppAndRegister(appId);
+ }
+ ContainerId containerId =
+ ContainerId.newContainerId(ApplicationAttemptId.newInstance(
+ appId, 1), 1);
+ RMContainer container = createRMContainer(containerId);
+ metricsPublisher.containerCreated(container, container.getCreationTime());
+ metricsPublisher.containerFinished(container, container.getFinishTime());
+ dispatcher.await();
+
+ String outputDirApp =
+ getTimelineEntityDir(app) + "/"
+ + TimelineEntityType.YARN_CONTAINER + "/";
+
+ File entityFolder = new File(outputDirApp);
+ Assert.assertTrue(entityFolder.isDirectory());
+
+ // file name is <entityId>.thist
+ String timelineServiceFileName =
+ containerId.toString()
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ File appFile = new File(outputDirApp, timelineServiceFileName);
+ Assert.assertTrue(appFile.exists());
+ Assert.assertEquals("Expected 2 events to be published", 2,
+ getNumOfNonEmptyLines(appFile));
+ }
+
+ private RMApp createAppAndRegister(ApplicationId appId) {
+ RMApp app = createRMApp(appId);
+
+ // some stuff which are currently taken care in RMAppImpl
+ rmAppsMapInContext.putIfAbsent(appId, app);
+ AppLevelTimelineCollector collector = new AppLevelTimelineCollector(appId);
+ rmTimelineCollectorManager.putIfAbsent(appId, collector);
+ return app;
+ }
+
+ private long getNumOfNonEmptyLines(File entityFile) throws IOException {
+ BufferedReader reader = null;
+ String strLine;
+ long count = 0;
+ try {
+ reader = new BufferedReader(new FileReader(entityFile));
+ while ((strLine = reader.readLine()) != null) {
+ if (strLine.trim().length() > 0)
+ count++;
+ }
+ } finally {
+ reader.close();
+ }
+ return count;
+ }
+
+ private String getTimelineEntityDir(RMApp app) {
+ String outputDirApp =
+ testRootDir.getAbsolutePath()+"/"
+ + FileSystemTimelineWriterImpl.ENTITIES_DIR
+ + "/"
+ + YarnConfiguration.DEFAULT_RM_CLUSTER_ID
+ + "/"
+ + app.getUser()
+ + "/"
+ + TimelineUtils.generateDefaultFlowIdBasedOnAppId(app
+ .getApplicationId()) + "/" + DEFAULT_FLOW_VERSION + "/"
+ + DEFAULT_FLOW_RUN + "/" + app.getApplicationId();
+ return outputDirApp;
+ }
+
+ private static RMApp createRMApp(ApplicationId appId) {
+ RMApp app = mock(RMAppImpl.class);
+ when(app.getApplicationId()).thenReturn(appId);
+ when(app.getName()).thenReturn("test app");
+ when(app.getApplicationType()).thenReturn("test app type");
+ when(app.getUser()).thenReturn("testUser");
+ when(app.getQueue()).thenReturn("test queue");
+ when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
+ when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
+ when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
+ when(app.getDiagnostics()).thenReturn(
+ new StringBuilder("test diagnostics info"));
+ RMAppAttempt appAttempt = mock(RMAppAttempt.class);
+ when(appAttempt.getAppAttemptId()).thenReturn(
+ ApplicationAttemptId.newInstance(appId, 1));
+ when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
+ when(app.getFinalApplicationStatus()).thenReturn(
+ FinalApplicationStatus.UNDEFINED);
+ when(app.getRMAppMetrics()).thenReturn(
+ new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
+ when(app.getApplicationTags()).thenReturn(Collections.<String> emptySet());
+ return app;
+ }
+
+ private static RMAppAttempt createRMAppAttempt(
+ ApplicationAttemptId appAttemptId) {
+ RMAppAttempt appAttempt = mock(RMAppAttempt.class);
+ when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId);
+ when(appAttempt.getHost()).thenReturn("test host");
+ when(appAttempt.getRpcPort()).thenReturn(-100);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(
+ ContainerId.newContainerId(appAttemptId, 1));
+ when(appAttempt.getMasterContainer()).thenReturn(container);
+ when(appAttempt.getDiagnostics()).thenReturn("test diagnostics info");
+ when(appAttempt.getTrackingUrl()).thenReturn("test tracking url");
+ when(appAttempt.getOriginalTrackingUrl()).thenReturn(
+ "test original tracking url");
+ return appAttempt;
+ }
+
+ private static RMContainer createRMContainer(ContainerId containerId) {
+ RMContainer container = mock(RMContainer.class);
+ when(container.getContainerId()).thenReturn(containerId);
+ when(container.getAllocatedNode()).thenReturn(
+ NodeId.newInstance("test host", -100));
+ when(container.getAllocatedResource()).thenReturn(
+ Resource.newInstance(-1, -1));
+ when(container.getAllocatedPriority()).thenReturn(Priority.UNDEFINED);
+ when(container.getCreationTime()).thenReturn(Integer.MAX_VALUE + 1L);
+ when(container.getFinishTime()).thenReturn(Integer.MAX_VALUE + 2L);
+ when(container.getDiagnosticsInfo()).thenReturn("test diagnostics info");
+ when(container.getContainerExitStatus()).thenReturn(-1);
+ when(container.getContainerState()).thenReturn(ContainerState.COMPLETE);
+ Container mockContainer = mock(Container.class);
+ when(container.getContainer()).thenReturn(mockContainer);
+ when(mockContainer.getNodeHttpAddress())
+ .thenReturn("http://localhost:1234");
+ return container;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index ee1515d..34a6b7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -55,7 +55,7 @@ public class FileSystemTimelineWriterImpl extends AbstractService
public static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
= "/tmp/timeline_service_data";
- private static final String ENTITIES_DIR = "entities";
+ public static final String ENTITIES_DIR = "entities";
/** Default extension for output files */
public static final String TIMELINE_SERVICE_STORAGE_EXTENSION = ".thist";
@@ -76,7 +76,7 @@ public class FileSystemTimelineWriterImpl extends AbstractService
return response;
}
- private void write(String clusterId, String userId, String flowName,
+ private synchronized void write(String clusterId, String userId, String flowName,
String flowVersion, long flowRun, String appId, TimelineEntity entity,
TimelineWriteResponse response) throws IOException {
PrintWriter out = null;
[47/50] [abbrv] hadoop git commit: YARN-4210. HBase reader throws NPE
if Get returns no rows (Varun Saxena via vrushali)
Posted by sj...@apache.org.
YARN-4210. HBase reader throws NPE if Get returns no rows (Varun Saxena via vrushali)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/529de2e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/529de2e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/529de2e1
Branch: refs/heads/YARN-2928-rebase
Commit: 529de2e17d9bf5bf5d7010d0680ad57f319fd71b
Parents: e4190b4
Author: Vrushali Channapattan <vr...@apache.org>
Authored: Wed Sep 30 13:56:07 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 17:05:05 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../storage/ApplicationEntityReader.java | 21 ++--------
.../storage/FlowActivityEntityReader.java | 27 +++++++------
.../storage/FlowRunEntityReader.java | 11 +++---
.../storage/GenericEntityReader.java | 3 +-
.../storage/TimelineEntityReader.java | 37 ++++++++++++------
.../TestTimelineReaderWebServicesFlowRun.java | 40 ++++++++++++++++++++
.../storage/TestHBaseTimelineStorage.java | 13 -------
8 files changed, 96 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/529de2e1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8cba705..066b243 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -115,6 +115,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-4203. Add request/response logging & timing for each REST endpoint
call (Varun Saxena via vrushali)
+ YARN-4210. HBase reader throws NPE if Get returns no rows (Varun Saxena
+ via vrushali)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/529de2e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
index dfbc31d..d5b5d63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
@@ -85,24 +86,10 @@ class ApplicationEntityReader extends GenericEntityReader {
}
@Override
- protected Iterable<Result> getResults(Configuration hbaseConf,
+ protected ResultScanner getResults(Configuration hbaseConf,
Connection conn) throws IOException {
- // If getEntities() is called for an application, there can be at most
- // one entity. If the entity passes the filter, it is returned. Otherwise,
- // an empty set is returned.
- byte[] rowKey = ApplicationRowKey.getRowKey(clusterId, userId, flowId,
- flowRunId, appId);
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- Result result = table.getResult(hbaseConf, conn, get);
- TimelineEntity entity = parseEntity(result);
- Set<Result> set;
- if (entity != null) {
- set = Collections.singleton(result);
- } else {
- set = Collections.emptySet();
- }
- return set;
+ throw new UnsupportedOperationException(
+ "we don't support multiple apps query");
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/529de2e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
index d5ece2e..e68ca17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
@@ -27,6 +27,7 @@ import java.util.TreeSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
@@ -88,18 +89,22 @@ class FlowActivityEntityReader extends TimelineEntityReader {
augmentParams(hbaseConf, conn);
NavigableSet<TimelineEntity> entities = new TreeSet<>();
- Iterable<Result> results = getResults(hbaseConf, conn);
- for (Result result : results) {
- TimelineEntity entity = parseEntity(result);
- if (entity == null) {
- continue;
- }
- entities.add(entity);
- if (entities.size() == limit) {
- break;
+ ResultScanner results = getResults(hbaseConf, conn);
+ try {
+ for (Result result : results) {
+ TimelineEntity entity = parseEntity(result);
+ if (entity == null) {
+ continue;
+ }
+ entities.add(entity);
+ if (entities.size() == limit) {
+ break;
+ }
}
+ return entities;
+ } finally {
+ results.close();
}
- return entities;
}
@Override
@@ -123,7 +128,7 @@ class FlowActivityEntityReader extends TimelineEntityReader {
}
@Override
- protected Iterable<Result> getResults(Configuration hbaseConf,
+ protected ResultScanner getResults(Configuration hbaseConf,
Connection conn) throws IOException {
Scan scan = new Scan();
scan.setRowPrefixFilter(FlowActivityRowKey.getRowKeyPrefix(clusterId));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/529de2e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
index ced795d..b5d7ae5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
@@ -96,7 +97,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
}
@Override
- protected Iterable<Result> getResults(Configuration hbaseConf,
+ protected ResultScanner getResults(Configuration hbaseConf,
Connection conn) throws IOException {
throw new UnsupportedOperationException(
"multiple entity query is not supported");
@@ -110,14 +111,14 @@ class FlowRunEntityReader extends TimelineEntityReader {
flowRun.setRunId(flowRunId);
// read the start time
- Long startTime = (Long)FlowRunColumn.MIN_START_TIME.readResult(result);
+ Number startTime = (Number)FlowRunColumn.MIN_START_TIME.readResult(result);
if (startTime != null) {
- flowRun.setStartTime(startTime);
+ flowRun.setStartTime(startTime.longValue());
}
// read the end time if available
- Long endTime = (Long)FlowRunColumn.MAX_END_TIME.readResult(result);
+ Number endTime = (Number)FlowRunColumn.MAX_END_TIME.readResult(result);
if (endTime != null) {
- flowRun.setMaxEndTime(endTime);
+ flowRun.setMaxEndTime(endTime.longValue());
}
// read the flow version
http://git-wip-us.apache.org/repos/asf/hadoop/blob/529de2e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
index 466914b..396a02b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -176,7 +177,7 @@ class GenericEntityReader extends TimelineEntityReader {
}
@Override
- protected Iterable<Result> getResults(Configuration hbaseConf,
+ protected ResultScanner getResults(Configuration hbaseConf,
Connection conn) throws IOException {
// Scan through part of the table to find the entities belong to one app
// and one type
http://git-wip-us.apache.org/repos/asf/hadoop/blob/529de2e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
index 0d1134c..93be2db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
@@ -25,9 +25,12 @@ import java.util.NavigableSet;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
@@ -40,6 +43,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
* entities that are being requested.
*/
abstract class TimelineEntityReader {
+ private static final Log LOG = LogFactory.getLog(TimelineEntityReader.class);
protected final boolean singleEntityRead;
protected String userId;
@@ -131,6 +135,11 @@ abstract class TimelineEntityReader {
augmentParams(hbaseConf, conn);
Result result = getResult(hbaseConf, conn);
+ if (result == null || result.isEmpty()) {
+ // Could not find a matching row.
+ LOG.info("Cannot find matching entity of type " + entityType);
+ return null;
+ }
return parseEntity(result);
}
@@ -145,18 +154,22 @@ abstract class TimelineEntityReader {
augmentParams(hbaseConf, conn);
NavigableSet<TimelineEntity> entities = new TreeSet<>();
- Iterable<Result> results = getResults(hbaseConf, conn);
- for (Result result : results) {
- TimelineEntity entity = parseEntity(result);
- if (entity == null) {
- continue;
- }
- entities.add(entity);
- if (entities.size() > limit) {
- entities.pollLast();
+ ResultScanner results = getResults(hbaseConf, conn);
+ try {
+ for (Result result : results) {
+ TimelineEntity entity = parseEntity(result);
+ if (entity == null) {
+ continue;
+ }
+ entities.add(entity);
+ if (entities.size() > limit) {
+ entities.pollLast();
+ }
}
+ return entities;
+ } finally {
+ results.close();
}
- return entities;
}
/**
@@ -184,9 +197,9 @@ abstract class TimelineEntityReader {
throws IOException;
/**
- * Fetches an iterator for {@link Result} instances for a multi-entity read.
+ * Fetches a {@link ResultScanner} for a multi-entity read.
*/
- protected abstract Iterable<Result> getResults(Configuration hbaseConf,
+ protected abstract ResultScanner getResults(Configuration hbaseConf,
Connection conn) throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/529de2e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java
index ae71e2c..e359f78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java
@@ -59,6 +59,7 @@ import org.junit.Test;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
@@ -281,6 +282,16 @@ public class TestTimelineReaderWebServicesFlowRun {
return false;
}
+ private static void verifyHttpResponse(Client client, URI uri,
+ Status status) {
+ ClientResponse resp =
+ client.resource(uri).accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertNotNull(resp);
+ assertTrue("Response from server should have been " + status,
+ resp.getClientResponseStatus().equals(status));
+ }
+
@Test
public void testGetFlowRun() throws Exception {
Client client = createClient();
@@ -354,6 +365,35 @@ public class TestTimelineReaderWebServicesFlowRun {
}
}
+ @Test
+ public void testGetFlowRunNotPresent() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowrun/cluster1/flow_name/1002345678929?userid=user1");
+ verifyHttpResponse(client, uri, Status.NOT_FOUND);
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetFlowsNotPresent() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flows/cluster2");
+ ClientResponse resp = getResponse(client, uri);
+ Set<FlowActivityEntity> entities =
+ resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
@After
public void stop() throws Exception {
if (server != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/529de2e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index 01920b3..3b0921b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -249,11 +249,7 @@ public class TestHBaseTimelineStorage {
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appId,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- appId, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
- assertEquals(1, es1.size());
// verify attributes
assertEquals(appId, e1.getId());
@@ -610,18 +606,9 @@ public class TestHBaseTimelineStorage {
TimelineEntity e2 = hbr.getEntity(user, cluster, null, null, appName,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es2 = hbr.getEntities(user, cluster, null, null,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
assertNotNull(e2);
assertEquals(e1, e2);
- assertEquals(1, es1.size());
- assertEquals(1, es2.size());
- assertEquals(es1, es2);
// check the events
NavigableSet<TimelineEvent> events = e1.getEvents();
[10/50] [abbrv] hadoop git commit: YARN-3721. build is broken on
YARN-2928 branch due to possible dependency cycle (Li Lu via sjlee)
Posted by sj...@apache.org.
YARN-3721. build is broken on YARN-2928 branch due to possible dependency cycle (Li Lu via sjlee)
(cherry picked from commit a9738ceb17b50cce8844fd42bb800c7f83f15caf)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59328637
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59328637
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59328637
Branch: refs/heads/YARN-2928-rebase
Commit: 59328637aa898d2173334daeb6eed82a67bdc017
Parents: 783f95f
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu May 28 12:03:53 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:56:40 2015 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 97 ++++++++++----------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop-yarn-server-timelineservice/pom.xml | 1 -
3 files changed, 53 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59328637/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 389381b..e42b2c9 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1001,55 +1001,58 @@
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${hbase.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-core</artifactId>
- <version>${phoenix.version}</version>
- <exclusions>
- <!-- Exclude jline from here -->
- <exclusion>
- <artifactId>jline</artifactId>
- <groupId>jline</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-core</artifactId>
- <type>test-jar</type>
- <version>${phoenix.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-it</artifactId>
- <version>${hbase.version}</version>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-testing-util</artifactId>
- <version>${hbase.version}</version>
- <scope>test</scope>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>org.jruby</groupId>
- <artifactId>jruby-complete</artifactId>
- </exclusion>
- <exclusion>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <version>${phoenix.version}</version>
+ <exclusions>
+ <!-- Exclude jline from here -->
+ <exclusion>
+ <artifactId>jline</artifactId>
+ <groupId>jline</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <type>test-jar</type>
+ <version>${phoenix.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.jruby</groupId>
+ <artifactId>jruby-complete</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59328637/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 87a24f1..e56403d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -79,6 +79,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3726. Fix TestHBaseTimelineWriterImpl unit test failure by fixing its
test data (Vrushali C via sjlee)
+ YARN-3721. build is broken on YARN-2928 branch due to possible dependency
+ cycle (Li Lu via sjlee)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59328637/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index 1e914de..da7fadf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -159,7 +159,6 @@
<scope>test</scope>
<optional>true</optional>
</dependency>
-
</dependencies>
<build>
[25/50] [abbrv] hadoop git commit: YARN-3984. Adjusted the event
column key schema and avoided missing empty event. Contributed by Vrushali C.
Posted by sj...@apache.org.
YARN-3984. Adjusted the event column key schema and avoided missing empty event. Contributed by Vrushali C.
(cherry picked from commit 895ccfa1ab9e701f2908586e323249f670fe5544)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45ce22f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45ce22f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45ce22f9
Branch: refs/heads/YARN-2928-rebase
Commit: 45ce22f917625f6318b0053863d6cdc7882e34d2
Parents: fb57d79
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Aug 5 16:28:57 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:40 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../storage/HBaseTimelineWriterImpl.java | 23 +++-
.../storage/common/TimelineWriterUtils.java | 13 +++
.../storage/entity/EntityRowKey.java | 18 +---
.../storage/TestHBaseTimelineWriterImpl.java | 105 ++++++++++++++++---
5 files changed, 128 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45ce22f9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b854cca..2cd44ad 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -121,6 +121,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3993. Changed to use the AM flag in ContainerContext determine AM
container in TestPerNodeTimelineCollectorsAuxService. (Sunil G via zjshen)
+ YARN-3984. Adjusted the event column key schema and avoided missing empty
+ event. (Vrushali C via zjshen)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45ce22f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index cd2e76e..3173e87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
@@ -200,20 +201,32 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
"! Using the current timestamp");
eventTimestamp = System.currentTimeMillis();
}
+ byte[] columnQualifierFirst =
+ Bytes.toBytes(Separator.VALUES.encode(eventId));
+ byte[] columnQualifierWithTsBytes =
+ Separator.VALUES.join(columnQualifierFirst,
+ Bytes.toBytes(TimelineWriterUtils.invert(eventTimestamp)));
Map<String, Object> eventInfo = event.getInfo();
- if (eventInfo != null) {
+ if ((eventInfo == null) || (eventInfo.size() == 0)) {
+ // add separator since event key is empty
+ byte[] compoundColumnQualifierBytes =
+ Separator.VALUES.join(columnQualifierWithTsBytes,
+ null);
+ String compoundColumnQualifier =
+ Bytes.toString(compoundColumnQualifierBytes);
+ EntityColumnPrefix.EVENT.store(rowKey, entityTable,
+ compoundColumnQualifier, null, TimelineWriterUtils.EMPTY_BYTES);
+ } else {
for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
// eventId?infoKey
- byte[] columnQualifierFirst =
- Bytes.toBytes(Separator.VALUES.encode(eventId));
byte[] compoundColumnQualifierBytes =
- Separator.VALUES.join(columnQualifierFirst,
+ Separator.VALUES.join(columnQualifierWithTsBytes,
Bytes.toBytes(info.getKey()));
// convert back to string to avoid additional API on store.
String compoundColumnQualifier =
Bytes.toString(compoundColumnQualifierBytes);
EntityColumnPrefix.EVENT.store(rowKey, entityTable,
- compoundColumnQualifier, eventTimestamp, info.getValue());
+ compoundColumnQualifier, null, info.getValue());
} // for info: eventInfo
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45ce22f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
index 28a0b6a..c957bf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
@@ -124,4 +124,17 @@ public class TimelineWriterUtils {
return segments;
}
+ /**
+ * Converts a timestamp into it's inverse timestamp to be used in (row) keys
+ * where we want to have the most recent timestamp in the top of the table
+ * (scans start at the most recent timestamp first).
+ *
+ * @param key value to be inverted so that the latest version will be first in
+ * a scan.
+ * @return inverted long
+ */
+ public static long invert(Long key) {
+ return Long.MAX_VALUE - key;
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45ce22f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
index 61958c2..3e17ad0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
/**
* Represents a rowkey for the entity table.
@@ -47,7 +48,7 @@ public class EntityRowKey {
flowId));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
- byte[] second = Bytes.toBytes(EntityRowKey.invert(flowRunId));
+ byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
byte[] third = Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId));
return Separator.QUALIFIERS.join(first, second, third);
}
@@ -70,24 +71,11 @@ public class EntityRowKey {
flowId));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
- byte[] second = Bytes.toBytes(EntityRowKey.invert(flowRunId));
+ byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
byte[] third =
Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, te.getType(),
te.getId()));
return Separator.QUALIFIERS.join(first, second, third);
}
- /**
- * Converts a timestamp into it's inverse timestamp to be used in (row) keys
- * where we want to have the most recent timestamp in the top of the table
- * (scans start at the most recent timestamp first).
- *
- * @param key value to be inverted so that the latest version will be first in
- * a scan.
- * @return inverted long
- */
- public static long invert(Long key) {
- return Long.MAX_VALUE - key;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45ce22f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index 31cb5d2..fd5643d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -43,8 +43,8 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
@@ -274,7 +274,7 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(user, Bytes.toString(rowKeyComponents[0]));
assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(EntityRowKey.invert(runid), Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(TimelineWriterUtils.invert(runid), Bytes.toLong(rowKeyComponents[3]));
assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
@@ -317,7 +317,6 @@ public class TestHBaseTimelineWriterImpl {
byte[] startRow =
EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
s.setStartRow(startRow);
- s.setMaxVersions(Integer.MAX_VALUE);
Connection conn = ConnectionFactory.createConnection(c1);
ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
@@ -331,24 +330,23 @@ public class TestHBaseTimelineWriterImpl {
assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
entity));
- // check the events
- NavigableMap<String, NavigableMap<Long, Object>> eventsResult =
- EntityColumnPrefix.EVENT.readResultsWithTimestamps(result);
+ Map<String, Object> eventsResult =
+ EntityColumnPrefix.EVENT.readResults(result);
// there should be only one event
assertEquals(1, eventsResult.size());
// key name for the event
- String valueKey = eventId + Separator.VALUES.getValue() + expKey;
- for (Map.Entry<String, NavigableMap<Long, Object>> e :
+ byte[] compoundColumnQualifierBytes =
+ Separator.VALUES.join(Bytes.toBytes(eventId),
+ Bytes.toBytes(TimelineWriterUtils.invert(expTs)),
+ Bytes.toBytes(expKey));
+ String valueKey = Bytes.toString(compoundColumnQualifierBytes);
+ for (Map.Entry<String, Object> e :
eventsResult.entrySet()) {
// the value key must match
assertEquals(valueKey, e.getKey());
- NavigableMap<Long, Object> value = e.getValue();
+ Object value = e.getValue();
// there should be only one timestamp and value
- assertEquals(1, value.size());
- for (Map.Entry<Long, Object> e2: value.entrySet()) {
- assertEquals(expTs, e2.getKey());
- assertEquals(expVal, e2.getValue());
- }
+ assertEquals(expVal, value.toString());
}
}
}
@@ -360,6 +358,85 @@ public class TestHBaseTimelineWriterImpl {
}
}
+ @Test
+ public void testAdditionalEntityEmptyEventInfo() throws IOException {
+ TimelineEvent event = new TimelineEvent();
+ String eventId = "foo_event_id";
+ event.setId(eventId);
+ Long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId("attempt_1329348432655_0001_m_000008_18");
+ entity.setType("FOO_ATTEMPT");
+ entity.addEvent(event);
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ String cluster = "cluster_emptyeventkey";
+ String user = "user_emptyeventkey";
+ String flow = "other_flow_name";
+ String flowVersion = "1111F01C2287BA";
+ long runid = 1009876543218L;
+ String appName = "some app name";
+ byte[] startRow =
+ EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
+ hbi.stop();
+ // scan the table and see that entity exists
+ Scan s = new Scan();
+ s.setStartRow(startRow);
+ s.addFamily(EntityColumnFamily.INFO.getBytes());
+ Connection conn = ConnectionFactory.createConnection(c1);
+ ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+
+ int rowCount = 0;
+ for (Result result : scanner) {
+ if (result != null && !result.isEmpty()) {
+ rowCount++;
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
+ entity));
+
+ Map<String, Object> eventsResult =
+ EntityColumnPrefix.EVENT.readResults(result);
+ // there should be only one event
+ assertEquals(1, eventsResult.size());
+ // key name for the event
+ byte[] compoundColumnQualifierWithTsBytes =
+ Separator.VALUES.join(Bytes.toBytes(eventId),
+ Bytes.toBytes(TimelineWriterUtils.invert(expTs)));
+ byte[] compoundColumnQualifierBytes =
+ Separator.VALUES.join(compoundColumnQualifierWithTsBytes,
+ null);
+ String valueKey = Bytes.toString(compoundColumnQualifierBytes);
+ for (Map.Entry<String, Object> e :
+ eventsResult.entrySet()) {
+ // the column qualifier key must match
+ assertEquals(valueKey, e.getKey());
+ Object value = e.getValue();
+ // value should be empty
+ assertEquals("", value.toString());
+ }
+ }
+ }
+ assertEquals(1, rowCount);
+
+ } finally {
+ hbi.stop();
+ hbi.close();
+ }
+ }
+
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
util.shutdownMiniCluster();
[11/50] [abbrv] hadoop git commit: YARN-3726. Fix
TestHBaseTimelineWriterImpl unit test failure by fixing its test data
(Vrushali C via sjlee)
Posted by sj...@apache.org.
YARN-3726. Fix TestHBaseTimelineWriterImpl unit test failure by fixing its test data (Vrushali C via sjlee)
(cherry picked from commit 4c0b6d73914f2e249795deb292f508177ea54884)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/783f95fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/783f95fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/783f95fc
Branch: refs/heads/YARN-2928-rebase
Commit: 783f95fc4f34fff5f7bcf2c19f3d77945defe31b
Parents: 6ac1fa7
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed May 27 20:28:04 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:56:40 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 4 ++++
.../storage/TestHBaseTimelineWriterImpl.java | 15 ++++++++-------
2 files changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/783f95fc/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 42ee90c..87a24f1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -75,6 +75,10 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3411. [Storage implementation] explore the native HBase write schema
for storage (Vrushali C via sjlee)
+
+ YARN-3726. Fix TestHBaseTimelineWriterImpl unit test failure by fixing its
+ test data (Vrushali C via sjlee)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/783f95fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index 48bacd6..f999b4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -121,12 +121,13 @@ public class TestHBaseTimelineWriterImpl {
TimelineMetric m1 = new TimelineMetric();
m1.setId("MAP_SLOT_MILLIS");
Map<Long, Number> metricValues = new HashMap<Long, Number>();
- metricValues.put(1429741609000L, 100000000);
- metricValues.put(1429742609000L, 200000000);
- metricValues.put(1429743609000L, 300000000);
- metricValues.put(1429744609000L, 400000000);
- metricValues.put(1429745609000L, 50000000000L);
- metricValues.put(1429746609000L, 60000000000L);
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 120000, 100000000);
+ metricValues.put(ts - 100000, 200000000);
+ metricValues.put(ts - 80000, 300000000);
+ metricValues.put(ts - 60000, 400000000);
+ metricValues.put(ts - 40000, 50000000000L);
+ metricValues.put(ts - 20000, 60000000000L);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
@@ -216,7 +217,7 @@ public class TestHBaseTimelineWriterImpl {
private void checkMetricsTimeseries(List<Cell> metricCells,
TimelineMetric m1) throws IOException {
Map<Long, Number> timeseries = m1.getValues();
- assertEquals(metricCells.size(), timeseries.size());
+ assertEquals(timeseries.size(), metricCells.size());
for (Cell c1 : metricCells) {
assertTrue(timeseries.containsKey(c1.getTimestamp()));
assertEquals(GenericObjectMapper.read(CellUtil.cloneValue(c1)),
[46/50] [abbrv] hadoop git commit: YARN-4203. Add request/response
logging & timing for each REST endpoint call (Varun Saxena via vrushali)
Posted by sj...@apache.org.
YARN-4203. Add request/response logging & timing for each REST endpoint call (Varun Saxena via vrushali)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4190b48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4190b48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4190b48
Branch: refs/heads/YARN-2928-rebase
Commit: e4190b489c3d54addd4a184bee389dc126b00e9a
Parents: ea801eb
Author: Vrushali Channapattan <vr...@apache.org>
Authored: Wed Sep 30 12:56:07 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 17:05:05 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../reader/TimelineReaderWebServices.java | 78 +++++++++++++++-----
2 files changed, 62 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4190b48/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 867c2aa..8cba705 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -112,6 +112,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-4075. [reader REST API] implement support for querying for flows
and flow runs (Varun Saxena via vrushali)
+ YARN-4203. Add request/response logging & timing for each REST endpoint
+ call (Varun Saxena via vrushali)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4190b48/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index f619c7d..a327099 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -44,6 +44,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
@@ -68,6 +69,7 @@ public class TimelineReaderWebServices {
private static final String COMMA_DELIMITER = ",";
private static final String COLON_DELIMITER = ":";
+ private static final String QUERY_STRING_SEP = "?";
private void init(HttpServletResponse response) {
response.setContentType(null);
@@ -185,14 +187,24 @@ public class TimelineReaderWebServices {
return callerUGI;
}
+ private static String getUserName(UserGroupInformation callerUGI) {
+ return ((callerUGI != null) ? callerUGI.getUserName().trim() : "");
+ }
+
private TimelineReaderManager getTimelineReaderManager() {
return (TimelineReaderManager)
ctxt.getAttribute(TimelineReaderServer.TIMELINE_READER_MANAGER_ATTR);
}
- private static void handleException(Exception e) throws BadRequestException,
+ private static void handleException(Exception e, String url, long startTime,
+ String invalidNumMsg) throws BadRequestException,
WebApplicationException {
- if (e instanceof IllegalArgumentException) {
+ long endTime = Time.monotonicNow();
+ LOG.info("Processed URL " + url + " but encountered exception (Took " +
+ (endTime - startTime) + " ms.)");
+ if (e instanceof NumberFormatException) {
+ throw new BadRequestException(invalidNumMsg + " is not a numeric value.");
+ } else if (e instanceof IllegalArgumentException) {
throw new BadRequestException("Requested Invalid Field.");
} else {
LOG.error("Error while processing REST request", e);
@@ -273,9 +285,14 @@ public class TimelineReaderWebServices {
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
@QueryParam("fields") String fields) {
+ String url = req.getRequestURI() +
+ (null == req.getQueryString() ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI = getUser(req);
+ LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
- UserGroupInformation callerUGI = getUser(req);
Set<TimelineEntity> entities = null;
try {
entities = timelineReaderManager.getEntities(
@@ -291,16 +308,16 @@ public class TimelineReaderWebServices {
parseValuesStr(metricfilters, COMMA_DELIMITER),
parseValuesStr(eventfilters, COMMA_DELIMITER),
parseFieldsStr(fields, COMMA_DELIMITER));
- } catch (NumberFormatException e) {
- throw new BadRequestException(
- "createdTime or modifiedTime start/end or limit or flowId is not" +
- " a numeric value.");
} catch (Exception e) {
- handleException(e);
+ handleException(e, url, startTime,
+ "createdTime or modifiedTime start/end or limit or flowId");
}
+ long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
}
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
return entities;
}
@@ -342,24 +359,32 @@ public class TimelineReaderWebServices {
@QueryParam("flowid") String flowId,
@QueryParam("flowrunid") String flowRunId,
@QueryParam("fields") String fields) {
+ String url = req.getRequestURI() +
+ (null == req.getQueryString() ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI = getUser(req);
+ LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
- UserGroupInformation callerUGI = getUser(req);
TimelineEntity entity = null;
try {
entity = timelineReaderManager.getEntity(
parseUser(callerUGI, userId), parseStr(clusterId), parseStr(flowId),
parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
parseStr(entityId), parseFieldsStr(fields, COMMA_DELIMITER));
- } catch (NumberFormatException e) {
- throw new BadRequestException("flowrunid is not a numeric value.");
} catch (Exception e) {
- handleException(e);
+ handleException(e, url, startTime, "flowrunid");
}
+ long endTime = Time.monotonicNow();
if (entity == null) {
+ LOG.info("Processed URL " + url + " but entity not found" + " (Took " +
+ (endTime - startTime) + " ms.)");
throw new NotFoundException("Timeline entity {id: " + parseStr(entityId) +
", type: " + parseStr(entityType) + " } is not found");
}
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
return entity;
}
@@ -394,9 +419,14 @@ public class TimelineReaderWebServices {
@PathParam("flowrunid") String flowRunId,
@QueryParam("userid") String userId,
@QueryParam("fields") String fields) {
+ String url = req.getRequestURI() +
+ (null == req.getQueryString() ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI = getUser(req);
+ LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
- UserGroupInformation callerUGI = getUser(req);
TimelineEntity entity = null;
try {
entity = timelineReaderManager.getEntity(
@@ -404,15 +434,18 @@ public class TimelineReaderWebServices {
parseStr(flowId), parseLongStr(flowRunId), null,
TimelineEntityType.YARN_FLOW_RUN.toString(), null,
parseFieldsStr(fields, COMMA_DELIMITER));
- } catch (NumberFormatException e) {
- throw new BadRequestException("flowRunId is not a numeric value.");
} catch (Exception e) {
- handleException(e);
+ handleException(e, url, startTime, "flowrunid");
}
+ long endTime = Time.monotonicNow();
if (entity == null) {
+ LOG.info("Processed URL " + url + " but flowrun not found (Took " +
+ (endTime - startTime) + " ms.)");
throw new NotFoundException("Flow run {flow id: " + parseStr(flowId) +
", run id: " + parseLongStr(flowRunId) + " } is not found");
}
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
return entity;
}
@@ -443,6 +476,12 @@ public class TimelineReaderWebServices {
@PathParam("clusterid") String clusterId,
@QueryParam("limit") String limit,
@QueryParam("fields") String fields) {
+ String url = req.getRequestURI() +
+ (null == req.getQueryString() ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI = getUser(req);
+ LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
@@ -452,14 +491,15 @@ public class TimelineReaderWebServices {
TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), parseLongStr(limit),
null, null, null, null, null, null, null, null, null, null,
parseFieldsStr(fields, COMMA_DELIMITER));
- } catch (NumberFormatException e) {
- throw new BadRequestException("limit is not a numeric value.");
} catch (Exception e) {
- handleException(e);
+ handleException(e, url, startTime, "limit");
}
+ long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
}
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
return entities;
}
}
\ No newline at end of file
[09/50] [abbrv] hadoop git commit: YARN-3634.
TestMRTimelineEventHandling and TestApplication are broken. Contributed by
Sangjin Lee.
Posted by sj...@apache.org.
YARN-3634. TestMRTimelineEventHandling and TestApplication are broken. Contributed by Sangjin Lee.
(cherry picked from commit b059dd4882fd759e4762cc11c019be4b68fb74c1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5be8b295
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5be8b295
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5be8b295
Branch: refs/heads/YARN-2928-rebase
Commit: 5be8b2951f0e37c263c4744627aae4ea02e8c3ce
Parents: 0478e1d
Author: Junping Du <ju...@apache.org>
Authored: Wed May 13 11:54:24 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:56:39 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../collectormanager/NMCollectorService.java | 5 +++
.../containermanager/ContainerManagerImpl.java | 2 +-
.../application/TestApplication.java | 3 +-
.../collector/NodeTimelineCollectorManager.java | 46 ++++++++++----------
5 files changed, 34 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5be8b295/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ee21af8..9ea0105 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -70,6 +70,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3529. Added mini HBase cluster and Phoenix support to timeline service
v2 unit tests. (Li Lu via zjshen)
+ YARN-3634. TestMRTimelineEventHandling and TestApplication are broken. (
+ Sangjin Lee via junping_du)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5be8b295/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index dc5601f..db79ee5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -81,6 +81,11 @@ public class NMCollectorService extends CompositeService implements
YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT));
server.start();
+ collectorServerAddress = conf.updateConnectAddr(
+ YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
+ server.getListenerAddress());
// start remaining services
super.serviceStart();
LOG.info("NMCollectorService started at " + collectorServerAddress);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5be8b295/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 616314e..da79446 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -933,7 +933,7 @@ public class ContainerManagerImpl extends CompositeService implements
TimelineUtils.FLOW_RUN_ID_TAG_PREFIX);
long flowRunId = 0L;
if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) {
- flowRunId = Long.valueOf(flowRunIdStr);
+ flowRunId = Long.parseLong(flowRunIdStr);
}
Application application = new ApplicationImpl(dispatcher, user,
flowName, flowVersion, flowRunId, applicationID, credentials, context);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5be8b295/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
index 3889d2e..002d4cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
@@ -519,7 +519,8 @@ public class TestApplication {
when(context.getApplicationACLsManager()).thenReturn(
new ApplicationACLsManager(conf));
when(context.getNMTokenSecretManager()).thenReturn(nmTokenSecretMgr);
-
+ when(context.getConf()).thenReturn(conf);
+
// Setting master key
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.setKeyId(123);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5be8b295/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
index 03ac770..31051db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
@@ -69,9 +69,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
private String timelineRestServerBindAddress;
- private CollectorNodemanagerProtocol nmCollectorService;
-
- private InetSocketAddress nmCollectorServiceAddress;
+ private volatile CollectorNodemanagerProtocol nmCollectorService;
static final String COLLECTOR_MANAGER_ATTR_KEY = "collector.manager";
@@ -85,18 +83,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
}
@Override
- public void serviceInit(Configuration conf) throws Exception {
- this.nmCollectorServiceAddress = conf.getSocketAddr(
- YarnConfiguration.NM_BIND_HOST,
- YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
- super.serviceInit(conf);
- }
-
- @Override
protected void serviceStart() throws Exception {
- nmCollectorService = getNMCollectorService();
startWebApp();
super.serviceStart();
}
@@ -176,7 +163,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
this.timelineRestServerBindAddress);
LOG.info("Report a new collector for application: " + appId +
" to the NM Collector Service.");
- nmCollectorService.reportNewCollectorInfo(request);
+ getNMCollectorService().reportNewCollectorInfo(request);
}
private void updateTimelineCollectorContext(
@@ -186,7 +173,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
GetTimelineCollectorContextRequest.newInstance(appId);
LOG.info("Get timeline collector context for " + appId);
GetTimelineCollectorContextResponse response =
- nmCollectorService.getTimelineCollectorContext(request);
+ getNMCollectorService().getTimelineCollectorContext(request);
String userId = response.getUserId();
if (userId != null && !userId.isEmpty()) {
collector.getTimelineEntityContext().setUserId(userId);
@@ -207,13 +194,26 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
@VisibleForTesting
protected CollectorNodemanagerProtocol getNMCollectorService() {
- Configuration conf = getConfig();
- final YarnRPC rpc = YarnRPC.create(conf);
-
- // TODO Security settings.
- return (CollectorNodemanagerProtocol) rpc.getProxy(
- CollectorNodemanagerProtocol.class,
- nmCollectorServiceAddress, conf);
+ if (nmCollectorService == null) {
+ synchronized (this) {
+ if (nmCollectorService == null) {
+ Configuration conf = getConfig();
+ InetSocketAddress nmCollectorServiceAddress = conf.getSocketAddr(
+ YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
+ LOG.info("nmCollectorServiceAddress: " + nmCollectorServiceAddress);
+ final YarnRPC rpc = YarnRPC.create(conf);
+
+ // TODO Security settings.
+ nmCollectorService = (CollectorNodemanagerProtocol) rpc.getProxy(
+ CollectorNodemanagerProtocol.class,
+ nmCollectorServiceAddress, conf);
+ }
+ }
+ }
+ return nmCollectorService;
}
@VisibleForTesting
[08/50] [abbrv] hadoop git commit: YARN-3411. [Storage
implementation] explore the native HBase write schema for storage (Vrushali C
via sjlee)
Posted by sj...@apache.org.
YARN-3411. [Storage implementation] explore the native HBase write schema for storage (Vrushali C via sjlee)
(cherry picked from commit 7a3068854d27eadae1c57545988f5b2029bf119a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ac1fa77
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ac1fa77
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ac1fa77
Branch: refs/heads/YARN-2928-rebase
Commit: 6ac1fa7797af4ef785cb037c2d3226a3d6e938a2
Parents: 5be8b29
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu May 21 14:11:01 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:56:39 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../collector/TimelineCollectorManager.java | 19 +
.../storage/EntityColumnDetails.java | 110 ++++++
.../storage/EntityColumnFamily.java | 95 +++++
.../storage/HBaseTimelineWriterImpl.java | 225 ++++++++++++
.../server/timelineservice/storage/Range.java | 59 ++++
.../storage/TimelineEntitySchemaConstants.java | 71 ++++
.../storage/TimelineSchemaCreator.java | 231 +++++++++++++
.../storage/TimelineWriterUtils.java | 344 +++++++++++++++++++
.../storage/TestHBaseTimelineWriterImpl.java | 292 ++++++++++++++++
10 files changed, 1448 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ac1fa77/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9ea0105..42ee90c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -73,6 +73,8 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3634. TestMRTimelineEventHandling and TestApplication are broken. (
Sangjin Lee via junping_du)
+ YARN-3411. [Storage implementation] explore the native HBase write schema
+ for storage (Vrushali C via sjlee)
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ac1fa77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 953d9b7..d54715c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -59,6 +59,13 @@ public abstract class TimelineCollectorManager extends AbstractService {
super.serviceInit(conf);
}
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ if (writer != null) {
+ writer.start();
+ }
+ }
// access to this map is synchronized with the map itself
private final Map<ApplicationId, TimelineCollector> collectors =
@@ -147,4 +154,16 @@ public abstract class TimelineCollectorManager extends AbstractService {
return collectors.containsKey(appId);
}
+ @Override
+ protected void serviceStop() throws Exception {
+ if (collectors != null && collectors.size() > 1) {
+ for (TimelineCollector c : collectors.values()) {
+ c.serviceStop();
+ }
+ }
+ if (writer != null) {
+ writer.close();
+ }
+ super.serviceStop();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ac1fa77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
new file mode 100644
index 0000000..2894c41
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Contains the Info Column Family details like Column names, types and byte
+ * representations for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * object that is stored in hbase Also has utility functions for storing each of
+ * these to the backend
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+enum EntityColumnDetails {
+ ID(EntityColumnFamily.INFO, "id"),
+ TYPE(EntityColumnFamily.INFO, "type"),
+ CREATED_TIME(EntityColumnFamily.INFO, "created_time"),
+ MODIFIED_TIME(EntityColumnFamily.INFO, "modified_time"),
+ FLOW_VERSION(EntityColumnFamily.INFO, "flow_version"),
+ PREFIX_IS_RELATED_TO(EntityColumnFamily.INFO, "r"),
+ PREFIX_RELATES_TO(EntityColumnFamily.INFO, "s"),
+ PREFIX_EVENTS(EntityColumnFamily.INFO, "e");
+
+ private final EntityColumnFamily columnFamily;
+ private final String value;
+ private final byte[] inBytes;
+
+ private EntityColumnDetails(EntityColumnFamily columnFamily,
+ String value) {
+ this.columnFamily = columnFamily;
+ this.value = value;
+ this.inBytes = Bytes.toBytes(this.value.toLowerCase());
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ byte[] getInBytes() {
+ return inBytes;
+ }
+
+ void store(byte[] rowKey, BufferedMutator entityTable, Object inputValue)
+ throws IOException {
+ TimelineWriterUtils.store(rowKey, entityTable,
+ this.columnFamily.getInBytes(), null, this.getInBytes(), inputValue,
+ null);
+ }
+
+ /**
+ * stores events data with column prefix
+ */
+ void store(byte[] rowKey, BufferedMutator entityTable, byte[] idBytes,
+ String key, Object inputValue) throws IOException {
+ TimelineWriterUtils.store(rowKey, entityTable,
+ this.columnFamily.getInBytes(),
+ // column prefix
+ TimelineWriterUtils.join(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
+ this.getInBytes(), idBytes),
+ // column qualifier
+ Bytes.toBytes(key),
+ inputValue, null);
+ }
+
+ /**
+ * stores relation entities with a column prefix
+ */
+ void store(byte[] rowKey, BufferedMutator entityTable, String key,
+ Set<String> inputValue) throws IOException {
+ TimelineWriterUtils.store(rowKey, entityTable,
+ this.columnFamily.getInBytes(),
+ // column prefix
+ this.getInBytes(),
+ // column qualifier
+ Bytes.toBytes(key),
+ // value
+ TimelineWriterUtils.getValueAsString(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR, inputValue),
+ // cell timestamp
+ null);
+ }
+
+ // TODO add a method that accepts a byte array,
+ // iterates over the enum and returns an enum from those bytes
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ac1fa77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
new file mode 100644
index 0000000..e556351
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Contains the Column family names and byte representations for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * object that is stored in hbase
+ * Also has utility functions for storing each of these to the backend
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+enum EntityColumnFamily {
+ INFO("i"),
+ CONFIG("c"),
+ METRICS("m");
+
+ private final String value;
+ private final byte[] inBytes;
+
+ private EntityColumnFamily(String value) {
+ this.value = value;
+ this.inBytes = Bytes.toBytes(this.value.toLowerCase());
+ }
+
+ byte[] getInBytes() {
+ return inBytes;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ /**
+ * stores the key as column and value as hbase column value in the given
+ * column family in the entity table
+ *
+ * @param rowKey
+ * @param entityTable
+ * @param inputValue
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, BufferedMutator entityTable, String key,
+ String inputValue) throws IOException {
+ if (key == null) {
+ return;
+ }
+ TimelineWriterUtils.store(rowKey, entityTable, this.getInBytes(), null,
+ Bytes.toBytes(key), inputValue, null);
+ }
+
+ /**
+ * stores the values along with cell timestamp
+ *
+ * @param rowKey
+ * @param entityTable
+ * @param key
+ * @param timestamp
+ * @param inputValue
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, BufferedMutator entityTable, String key,
+ Long timestamp, Number inputValue) throws IOException {
+ if (key == null) {
+ return;
+ }
+ TimelineWriterUtils.store(rowKey, entityTable, this.getInBytes(), null,
+ Bytes.toBytes(key), inputValue, timestamp);
+ }
+
+ // TODO add a method that accepts a byte array,
+ // iterates over the enum and returns an enum from those bytes
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ac1fa77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
new file mode 100644
index 0000000..aa71c6c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineEntitySchemaConstants;
+
+/**
+ * This implements a hbase based backend for storing application timeline entity
+ * information.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class HBaseTimelineWriterImpl extends AbstractService implements
+ TimelineWriter {
+
+ private Connection conn;
+ private BufferedMutator entityTable;
+
+ private static final Log LOG = LogFactory
+ .getLog(HBaseTimelineWriterImpl.class);
+
+ public HBaseTimelineWriterImpl() {
+ super(HBaseTimelineWriterImpl.class.getName());
+ }
+
+ public HBaseTimelineWriterImpl(Configuration conf) throws IOException {
+ super(conf.get("yarn.application.id",
+ HBaseTimelineWriterImpl.class.getName()));
+ }
+
+ /**
+ * initializes the hbase connection to write to the entity table
+ */
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ Configuration hbaseConf = HBaseConfiguration.create(conf);
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ TableName entityTableName = TableName.valueOf(hbaseConf.get(
+ TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
+ TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME));
+ entityTable = conn.getBufferedMutator(entityTableName);
+ }
+
+ /**
+ * Stores the entire information in TimelineEntities to the timeline store.
+ */
+ @Override
+ public TimelineWriteResponse write(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRunId, String appId,
+ TimelineEntities data) throws IOException {
+
+ byte[] rowKeyPrefix = TimelineWriterUtils.getRowKeyPrefix(clusterId,
+ userId, flowName, flowRunId, appId);
+
+ TimelineWriteResponse putStatus = new TimelineWriteResponse();
+ for (TimelineEntity te : data.getEntities()) {
+
+ // a set can have at most 1 null
+ if (te == null) {
+ continue;
+ }
+ // get row key
+ byte[] row = TimelineWriterUtils.join(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES, rowKeyPrefix,
+ Bytes.toBytes(te.getType()), Bytes.toBytes(te.getId()));
+
+ storeInfo(row, te, flowVersion);
+ storeEvents(row, te.getEvents());
+ storeConfig(row, te.getConfigs());
+ storeMetrics(row, te.getMetrics());
+ storeRelations(row, te.getIsRelatedToEntities(),
+ EntityColumnDetails.PREFIX_IS_RELATED_TO);
+ storeRelations(row, te.getRelatesToEntities(),
+ EntityColumnDetails.PREFIX_RELATES_TO);
+ }
+
+ return putStatus;
+ }
+
+ /**
+ * Stores the Relations from the {@linkplain TimelineEntity} object
+ */
+ private void storeRelations(byte[] rowKey,
+ Map<String, Set<String>> connectedEntities,
+ EntityColumnDetails columnNamePrefix) throws IOException {
+ for (Map.Entry<String, Set<String>> entry : connectedEntities.entrySet()) {
+ columnNamePrefix.store(rowKey, entityTable, entry.getKey(),
+ entry.getValue());
+ }
+ }
+
+ /**
+ * Stores information from the {@linkplain TimelineEntity} object
+ */
+ private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion)
+ throws IOException {
+
+ EntityColumnDetails.ID.store(rowKey, entityTable, te.getId());
+ EntityColumnDetails.TYPE.store(rowKey, entityTable, te.getType());
+ EntityColumnDetails.CREATED_TIME.store(rowKey, entityTable,
+ te.getCreatedTime());
+ EntityColumnDetails.MODIFIED_TIME.store(rowKey, entityTable,
+ te.getModifiedTime());
+ EntityColumnDetails.FLOW_VERSION.store(rowKey, entityTable, flowVersion);
+ }
+
+ /**
+ * stores the config information from {@linkplain TimelineEntity}
+ */
+ private void storeConfig(byte[] rowKey, Map<String, String> config)
+ throws IOException {
+ if (config == null) {
+ return;
+ }
+ for (Map.Entry<String, String> entry : config.entrySet()) {
+ EntityColumnFamily.CONFIG.store(rowKey, entityTable,
+ entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * stores the {@linkplain TimelineMetric} information from the
+ * {@linkplain TimelineEvent} object
+ */
+ private void storeMetrics(byte[] rowKey, Set<TimelineMetric> metrics)
+ throws IOException {
+ if (metrics != null) {
+ for (TimelineMetric metric : metrics) {
+ String key = metric.getId();
+ Map<Long, Number> timeseries = metric.getValues();
+ for (Map.Entry<Long, Number> entry : timeseries.entrySet()) {
+ EntityColumnFamily.METRICS.store(rowKey, entityTable, key,
+ entry.getKey(), entry.getValue());
+ }
+ }
+ }
+ }
+
+ /**
+ * Stores the events from the {@linkplain TimelineEvent} object
+ */
+ private void storeEvents(byte[] rowKey, Set<TimelineEvent> events)
+ throws IOException {
+ if (events != null) {
+ for (TimelineEvent event : events) {
+ if (event != null) {
+ String id = event.getId();
+ if (id != null) {
+ byte[] idBytes = Bytes.toBytes(id);
+ Map<String, Object> eventInfo = event.getInfo();
+ if (eventInfo != null) {
+ for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
+ EntityColumnDetails.PREFIX_EVENTS.store(rowKey,
+ entityTable, idBytes, info.getKey(), info.getValue());
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public TimelineWriteResponse aggregate(TimelineEntity data,
+ TimelineAggregationTrack track) throws IOException {
+ return null;
+ }
+
+ /**
+ * close the hbase connections
+ * The close APIs perform flushing and release any
+ * resources held
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ if (entityTable != null) {
+ LOG.info("closing entity table");
+ // The close API performs flushing and releases any resources held
+ entityTable.close();
+ }
+ if (conn != null) {
+ LOG.info("closing the hbase Connection");
+ conn.close();
+ }
+ super.serviceStop();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ac1fa77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
new file mode 100644
index 0000000..2a2db81
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Range {
+ private final int startIdx;
+ private final int endIdx;
+
+ /**
+ * Defines a range from start index (inclusive) to end index (exclusive).
+ *
+ * @param start
+ * Starting index position
+ * @param end
+ * Ending index position (exclusive)
+ */
+ public Range(int start, int end) {
+ if (start < 0 || end < start) {
+ throw new IllegalArgumentException(
+ "Invalid range, required that: 0 <= start <= end; start=" + start
+ + ", end=" + end);
+ }
+
+ this.startIdx = start;
+ this.endIdx = end;
+ }
+
+ public int start() {
+ return startIdx;
+ }
+
+ public int end() {
+ return endIdx;
+ }
+
+ public int length() {
+ return endIdx - startIdx;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ac1fa77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
new file mode 100644
index 0000000..d95cbb2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * contains the constants used in the context of schema accesses for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * information
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class TimelineEntitySchemaConstants {
+
+ /** entity prefix */
+ public static final String ENTITY_PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX
+ + ".entity";
+
+ /** config param name that specifies the entity table name */
+ public static final String ENTITY_TABLE_NAME = ENTITY_PREFIX
+ + ".table.name";
+
+ /**
+ * config param name that specifies the TTL for metrics column family in
+ * entity table
+ */
+ public static final String ENTITY_TABLE_METRICS_TTL = ENTITY_PREFIX
+ + ".table.metrics.ttl";
+
+ /** default value for entity table name */
+ public static final String DEFAULT_ENTITY_TABLE_NAME = "timelineservice.entity";
+
+ /** in bytes default value for entity table name */
+ static final byte[] DEFAULT_ENTITY_TABLE_NAME_BYTES = Bytes
+ .toBytes(DEFAULT_ENTITY_TABLE_NAME);
+
+ /** separator in row key */
+ public static final String ROW_KEY_SEPARATOR = "!";
+
+ /** byte representation of the separator in row key */
+ static final byte[] ROW_KEY_SEPARATOR_BYTES = Bytes
+ .toBytes(ROW_KEY_SEPARATOR);
+
+ public static final byte ZERO_BYTES = 0;
+
+ /** default TTL is 30 days for metrics timeseries */
+ public static final int ENTITY_TABLE_METRICS_TTL_DEFAULT = 2592000;
+
+ /** default max number of versions */
+ public static final int ENTITY_TABLE_METRICS_MAX_VERSIONS_DEFAULT = 1000;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ac1fa77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
new file mode 100644
index 0000000..820a6d1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -0,0 +1,231 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+/**
+ * This creates the schema for a hbase based backend for storing application
+ * timeline information.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class TimelineSchemaCreator {
+
+ final static String NAME = TimelineSchemaCreator.class.getSimpleName();
+ private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
+ final static byte[][] splits = { Bytes.toBytes("a"), Bytes.toBytes("ad"),
+ Bytes.toBytes("an"), Bytes.toBytes("b"), Bytes.toBytes("ca"),
+ Bytes.toBytes("cl"), Bytes.toBytes("d"), Bytes.toBytes("e"),
+ Bytes.toBytes("f"), Bytes.toBytes("g"), Bytes.toBytes("h"),
+ Bytes.toBytes("i"), Bytes.toBytes("j"), Bytes.toBytes("k"),
+ Bytes.toBytes("l"), Bytes.toBytes("m"), Bytes.toBytes("n"),
+ Bytes.toBytes("o"), Bytes.toBytes("q"), Bytes.toBytes("r"),
+ Bytes.toBytes("s"), Bytes.toBytes("se"), Bytes.toBytes("t"),
+ Bytes.toBytes("u"), Bytes.toBytes("v"), Bytes.toBytes("w"),
+ Bytes.toBytes("x"), Bytes.toBytes("y"), Bytes.toBytes("z") };
+
+ public static final String SPLIT_KEY_PREFIX_LENGTH = "4";
+
+ public static void main(String[] args) throws Exception {
+
+ Configuration hbaseConf = HBaseConfiguration.create();
+ // Grab input args and allow for -Dxyz style arguments
+ String[] otherArgs = new GenericOptionsParser(hbaseConf, args)
+ .getRemainingArgs();
+
+ // Grab the arguments we're looking for.
+ CommandLine commandLine = parseArgs(otherArgs);
+
+ // Grab the entityTableName argument
+ String entityTableName = commandLine.getOptionValue("e");
+ if (StringUtils.isNotBlank(entityTableName)) {
+ hbaseConf.set(TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
+ entityTableName);
+ }
+ String entityTable_TTL_Metrics = commandLine.getOptionValue("m");
+ if (StringUtils.isNotBlank(entityTable_TTL_Metrics)) {
+ hbaseConf.set(TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL,
+ entityTable_TTL_Metrics);
+ }
+ createAllTables(hbaseConf);
+ }
+
+ /**
+ * Parse command-line arguments.
+ *
+ * @param args
+ * command line arguments passed to program.
+ * @return parsed command line.
+ * @throws ParseException
+ */
+ private static CommandLine parseArgs(String[] args) throws ParseException {
+ Options options = new Options();
+
+ // Input
+ Option o = new Option("e", "entityTableName", true, "entity table name");
+ o.setArgName("entityTableName");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option("m", "metricsTTL", true, "TTL for metrics column family");
+ o.setArgName("metricsTTL");
+ o.setRequired(false);
+ options.addOption(o);
+
+ CommandLineParser parser = new PosixParser();
+ CommandLine commandLine = null;
+ try {
+ commandLine = parser.parse(options, args);
+ } catch (Exception e) {
+ LOG.error("ERROR: " + e.getMessage() + "\n");
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(NAME + " ", options, true);
+ System.exit(-1);
+ }
+
+ return commandLine;
+ }
+
+ private static void createAllTables(Configuration hbaseConf)
+ throws IOException {
+
+ Connection conn = null;
+ try {
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ Admin admin = conn.getAdmin();
+ if (admin == null) {
+ throw new IOException("Cannot create table since admin is null");
+ }
+ createTimelineEntityTable(admin, hbaseConf);
+ } finally {
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+ /**
+ * Creates a table with column families info, config and metrics
+ * info stores information about a timeline entity object
+ * config stores configuration data of a timeline entity object
+ * metrics stores the metrics of a timeline entity object
+ *
+ * Example entity table record:
+ * <pre>
+ *|------------------------------------------------------------|
+ *| Row | Column Family | Column Family | Column Family|
+ *| key | info | metrics | config |
+ *|------------------------------------------------------------|
+ *| userName! | id:entityId | metricName1: | configKey1: |
+ *| clusterId! | | metricValue1 | configValue1 |
+ *| flowId! | type:entityType| @timestamp1 | |
+ *| flowRunId! | | | configKey2: |
+ *| AppId! | created_time: | metricName1: | configValue2 |
+ *| entityType!| 1392993084018 | metricValue2 | |
+ *| entityId | | @timestamp2 | |
+ *| | modified_time: | | |
+ *| | 1392995081012 | metricName2: | |
+ *| | | metricValue1 | |
+ *| | r!relatesToKey:| @timestamp2 | |
+ *| | id3!id4!id5 | | |
+ *| | | | |
+ *| | s!isRelatedToKey| | |
+ *| | id7!id9!id5 | | |
+ *| | | | |
+ *| | e!eventKey: | | |
+ *| | eventValue | | |
+ *| | | | |
+ *| | flowVersion: | | |
+ *| | versionValue | | |
+ *|------------------------------------------------------------|
+ *</pre>
+ * @param admin
+ * @param hbaseConf
+ * @throws IOException
+ */
+ public static void createTimelineEntityTable(Admin admin,
+ Configuration hbaseConf) throws IOException {
+
+ TableName table = TableName.valueOf(hbaseConf.get(
+ TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
+ TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME));
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor entityTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor cf1 = new HColumnDescriptor(
+ EntityColumnFamily.INFO.getInBytes());
+ cf1.setBloomFilterType(BloomType.ROWCOL);
+ entityTableDescp.addFamily(cf1);
+
+ HColumnDescriptor cf2 = new HColumnDescriptor(
+ EntityColumnFamily.CONFIG.getInBytes());
+ cf2.setBloomFilterType(BloomType.ROWCOL);
+ cf2.setBlockCacheEnabled(true);
+ entityTableDescp.addFamily(cf2);
+
+ HColumnDescriptor cf3 = new HColumnDescriptor(
+ EntityColumnFamily.METRICS.getInBytes());
+ entityTableDescp.addFamily(cf3);
+ cf3.setBlockCacheEnabled(true);
+ // always keep 1 version (the latest)
+ cf3.setMinVersions(1);
+ cf3.setMaxVersions(TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_MAX_VERSIONS_DEFAULT);
+ cf3.setTimeToLive(hbaseConf.getInt(
+ TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL,
+ TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL_DEFAULT));
+ entityTableDescp
+ .setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(entityTableDescp, splits);
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ac1fa77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
new file mode 100644
index 0000000..113935e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.Range;
+
+/**
+ * bunch of utility functions used across TimelineWriter classes
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class TimelineWriterUtils {
+
+ /** empty bytes */
+ public static final byte[] EMPTY_BYTES = new byte[0];
+ private static final String SPACE = " ";
+ private static final String UNDERSCORE = "_";
+ private static final String EMPTY_STRING = "";
+
+ /**
+ * Returns a single byte array containing all of the individual component
+ * arrays separated by the separator array.
+ *
+ * @param separator
+ * @param components
+ * @return byte array after joining the components
+ */
+ public static byte[] join(byte[] separator, byte[]... components) {
+ if (components == null || components.length == 0) {
+ return EMPTY_BYTES;
+ }
+
+ int finalSize = 0;
+ if (separator != null) {
+ finalSize = separator.length * (components.length - 1);
+ }
+ for (byte[] comp : components) {
+ if (comp != null) {
+ finalSize += comp.length;
+ }
+ }
+
+ byte[] buf = new byte[finalSize];
+ int offset = 0;
+ for (int i = 0; i < components.length; i++) {
+ if (components[i] != null) {
+ System.arraycopy(components[i], 0, buf, offset, components[i].length);
+ offset += components[i].length;
+ if (i < (components.length - 1) && separator != null
+ && separator.length > 0) {
+ System.arraycopy(separator, 0, buf, offset, separator.length);
+ offset += separator.length;
+ }
+ }
+ }
+ return buf;
+ }
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments. To identify the split
+ * ranges without the array copies, see
+ * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
+ *
+ * @param source
+ * @param separator
+ * @return byte[] array after splitting the source
+ */
+ public static byte[][] split(byte[] source, byte[] separator) {
+ return split(source, separator, -1);
+ }
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments. To identify the split
+ * ranges without the array copies, see
+ * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
+ *
+ * @param source
+ * @param separator
+ * @param limit
+ * @return byte[][] after splitting the input source
+ */
+ public static byte[][] split(byte[] source, byte[] separator, int limit) {
+ List<Range> segments = splitRanges(source, separator, limit);
+
+ byte[][] splits = new byte[segments.size()][];
+ for (int i = 0; i < segments.size(); i++) {
+ Range r = segments.get(i);
+ byte[] tmp = new byte[r.length()];
+ if (tmp.length > 0) {
+ System.arraycopy(source, r.start(), tmp, 0, r.length());
+ }
+ splits[i] = tmp;
+ }
+ return splits;
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ */
+ public static List<Range> splitRanges(byte[] source, byte[] separator) {
+ return splitRanges(source, separator, -1);
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ * @param source the source data
+ * @param separator the separator pattern to look for
+ * @param limit the maximum number of splits to identify in the source
+ */
+ public static List<Range> splitRanges(byte[] source, byte[] separator, int limit) {
+ List<Range> segments = new ArrayList<Range>();
+ if ((source == null) || (separator == null)) {
+ return segments;
+ }
+ int start = 0;
+ itersource: for (int i = 0; i < source.length; i++) {
+ for (int j = 0; j < separator.length; j++) {
+ if (source[i + j] != separator[j]) {
+ continue itersource;
+ }
+ }
+ // all separator elements matched
+ if (limit > 0 && segments.size() >= (limit-1)) {
+ // everything else goes in one final segment
+ break;
+ }
+
+ segments.add(new Range(start, i));
+ start = i + separator.length;
+ // i will be incremented again in outer for loop
+ i += separator.length-1;
+ }
+ // add in remaining to a final range
+ if (start <= source.length) {
+ segments.add(new Range(start, source.length));
+ }
+ return segments;
+ }
+
+ /**
+ * converts run id into it's inverse timestamp
+ * @param flowRunId
+ * @return inverted long
+ */
+ public static long encodeRunId(Long flowRunId) {
+ return Long.MAX_VALUE - flowRunId;
+ }
+
+ /**
+ * return a value from the Map as a String
+ * @param key
+ * @param values
+ * @return value as a String or ""
+ * @throws IOException
+ */
+ public static String getValueAsString(final byte[] key,
+ final Map<byte[], byte[]> values) throws IOException {
+ if( values == null ) {
+ return EMPTY_STRING;
+ }
+ byte[] value = values.get(key);
+ if (value != null) {
+ return GenericObjectMapper.read(value).toString();
+ } else {
+ return EMPTY_STRING;
+ }
+ }
+
+ /**
+ * return a value from the Map as a long
+ * @param key
+ * @param values
+ * @return value as Long or 0L
+ * @throws IOException
+ */
+ public static long getValueAsLong(final byte[] key,
+ final Map<byte[], byte[]> values) throws IOException {
+ if (values == null) {
+ return 0;
+ }
+ byte[] value = values.get(key);
+ if (value != null) {
+ Number val = (Number) GenericObjectMapper.read(value);
+ return val.longValue();
+ } else {
+ return 0L;
+ }
+ }
+
+ /**
+ * concates the values from a Set<Strings> to return a single delimited string value
+ * @param rowKeySeparator
+ * @param values
+ * @return Value from the set of strings as a string
+ */
+ public static String getValueAsString(String rowKeySeparator,
+ Set<String> values) {
+
+ if (values == null) {
+ return EMPTY_STRING;
+ }
+ StringBuilder concatStrings = new StringBuilder();
+ for (String value : values) {
+ concatStrings.append(value);
+ concatStrings.append(rowKeySeparator);
+ }
+ // remove the last separator
+ if(concatStrings.length() > 1) {
+ concatStrings.deleteCharAt(concatStrings.lastIndexOf(rowKeySeparator));
+ }
+ return concatStrings.toString();
+ }
+ /**
+ * Constructs a row key prefix for the entity table
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @param appId
+ * @return byte array with the row key prefix
+ */
+ static byte[] getRowKeyPrefix(String clusterId, String userId, String flowId,
+ Long flowRunId, String appId) {
+ return TimelineWriterUtils.join(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
+ Bytes.toBytes(cleanse(userId)), Bytes.toBytes(cleanse(clusterId)),
+ Bytes.toBytes(cleanse(flowId)),
+ Bytes.toBytes(TimelineWriterUtils.encodeRunId(flowRunId)),
+ Bytes.toBytes(cleanse(appId)));
+ }
+
+ /**
+ * Takes a string token to be used as a key or qualifier and
+ * cleanses out reserved tokens.
+ * This operation is not symmetrical.
+ * Logic is to replace all spaces and separator chars in input with
+ * underscores.
+ *
+ * @param token token to cleanse.
+ * @return String with no spaces and no separator chars
+ */
+ public static String cleanse(String token) {
+ if (token == null || token.length() == 0) {
+ return token;
+ }
+
+ String cleansed = token.replaceAll(SPACE, UNDERSCORE);
+ cleansed = cleansed.replaceAll(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR, UNDERSCORE);
+
+ return cleansed;
+ }
+
+ /**
+ * stores the info to the table in hbase
+ *
+ * @param rowKey
+ * @param table
+ * @param columnFamily
+ * @param columnPrefix
+ * @param columnQualifier
+ * @param inputValue
+ * @param cellTimeStamp
+ * @throws IOException
+ */
+ public static void store(byte[] rowKey, BufferedMutator table, byte[] columnFamily,
+ byte[] columnPrefix, byte[] columnQualifier, Object inputValue,
+ Long cellTimeStamp) throws IOException {
+ if ((rowKey == null) || (table == null) || (columnFamily == null)
+ || (columnQualifier == null) || (inputValue == null)) {
+ return;
+ }
+
+ Put p = null;
+ if (cellTimeStamp == null) {
+ if (columnPrefix != null) {
+ // store with prefix
+ p = new Put(rowKey);
+ p.addColumn(
+ columnFamily,
+ join(TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
+ columnPrefix, columnQualifier), GenericObjectMapper
+ .write(inputValue));
+ } else {
+ // store without prefix
+ p = new Put(rowKey);
+ p.addColumn(columnFamily, columnQualifier,
+ GenericObjectMapper.write(inputValue));
+ }
+ } else {
+ // store with cell timestamp
+ Cell cell = CellUtil.createCell(rowKey, columnFamily, columnQualifier,
+ // set the cell timestamp
+ cellTimeStamp,
+ // KeyValue Type minimum
+ TimelineEntitySchemaConstants.ZERO_BYTES,
+ GenericObjectMapper.write(inputValue));
+ p = new Put(rowKey);
+ p.add(cell);
+ }
+ if (p != null) {
+ table.mutate(p);
+ }
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ac1fa77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
new file mode 100644
index 0000000..48bacd6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -0,0 +1,292 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import org.junit.Test;
+
+/**
+ * Unit test HBaseTimelineWriterImpl
+ * YARN 3411
+ *
+ * @throws Exception
+ */
+public class TestHBaseTimelineWriterImpl {
+
+ private static HBaseTestingUtility util;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ util = new HBaseTestingUtility();
+ util.startMiniCluster();
+ createSchema();
+ }
+
+ private static void createSchema() throws IOException {
+ byte[][] families = new byte[3][];
+ families[0] = EntityColumnFamily.INFO.getInBytes();
+ families[1] = EntityColumnFamily.CONFIG.getInBytes();
+ families[2] = EntityColumnFamily.METRICS.getInBytes();
+ TimelineSchemaCreator.createTimelineEntityTable(util.getHBaseAdmin(),
+ util.getConfiguration());
+ }
+
+ @Test
+ public void testWriteEntityToHBase() throws Exception {
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entity = new TimelineEntity();
+ String id = "hello";
+ String type = "world";
+ entity.setId(id);
+ entity.setType(type);
+ Long cTime = 1425016501000L;
+ Long mTime = 1425026901000L;
+ entity.setCreatedTime(cTime);
+ entity.setModifiedTime(mTime);
+
+ // add the isRelatedToEntity info
+ String key = "task";
+ String value = "is_related_to_entity_id_here";
+ Set<String> isRelatedToSet = new HashSet<String>();
+ isRelatedToSet.add(value);
+ Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
+ isRelatedTo.put(key, isRelatedToSet);
+ entity.setIsRelatedToEntities(isRelatedTo);
+
+ // add the relatesTo info
+ key = "container";
+ value = "relates_to_entity_id_here";
+ Set<String> relatesToSet = new HashSet<String>();
+ relatesToSet.add(value);
+ value = "relates_to_entity_id_here_Second";
+ relatesToSet.add(value);
+ Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
+ relatesTo.put(key, relatesToSet);
+ entity.setRelatesToEntities(relatesTo);
+
+ // add some config entries
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put("config_param1", "value1");
+ conf.put("config_param2", "value2");
+ entity.addConfigs(conf);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ metricValues.put(1429741609000L, 100000000);
+ metricValues.put(1429742609000L, 200000000);
+ metricValues.put(1429743609000L, 300000000);
+ metricValues.put(1429744609000L, 400000000);
+ metricValues.put(1429745609000L, 50000000000L);
+ metricValues.put(1429746609000L, 60000000000L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ te.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ String cluster = "cluster1";
+ String user = "user1";
+ String flow = "some_flow_name";
+ String flowVersion = "AB7822C10F1111";
+ long runid = 1002345678919L;
+ String appName = "some app name";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+ hbi.stop();
+
+ // scan the table and see that entity exists
+ Scan s = new Scan();
+ byte[] startRow = TimelineWriterUtils.getRowKeyPrefix(cluster, user, flow,
+ runid, appName);
+ s.setStartRow(startRow);
+ s.setMaxVersions(Integer.MAX_VALUE);
+ ResultScanner scanner = null;
+ TableName entityTableName = TableName
+ .valueOf(TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Table entityTable = conn.getTable(entityTableName);
+ int rowCount = 0;
+ int colCount = 0;
+ scanner = entityTable.getScanner(s);
+ for (Result result : scanner) {
+ if (result != null && !result.isEmpty()) {
+ rowCount++;
+ colCount += result.size();
+ byte[] row1 = result.getRow();
+ assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
+ entity));
+
+ // check info column family
+ NavigableMap<byte[], byte[]> infoValues = result
+ .getFamilyMap(EntityColumnFamily.INFO.getInBytes());
+ String id1 = TimelineWriterUtils.getValueAsString(
+ EntityColumnDetails.ID.getInBytes(), infoValues);
+ assertEquals(id, id1);
+ String type1 = TimelineWriterUtils.getValueAsString(
+ EntityColumnDetails.TYPE.getInBytes(), infoValues);
+ assertEquals(type, type1);
+ Long cTime1 = TimelineWriterUtils.getValueAsLong(
+ EntityColumnDetails.CREATED_TIME.getInBytes(), infoValues);
+ assertEquals(cTime1, cTime);
+ Long mTime1 = TimelineWriterUtils.getValueAsLong(
+ EntityColumnDetails.MODIFIED_TIME.getInBytes(), infoValues);
+ assertEquals(mTime1, mTime);
+ checkRelatedEntities(isRelatedTo, infoValues,
+ EntityColumnDetails.PREFIX_IS_RELATED_TO.getInBytes());
+ checkRelatedEntities(relatesTo, infoValues,
+ EntityColumnDetails.PREFIX_RELATES_TO.getInBytes());
+
+ // check config column family
+ NavigableMap<byte[], byte[]> configValuesResult = result
+ .getFamilyMap(EntityColumnFamily.CONFIG.getInBytes());
+ checkConfigs(configValuesResult, conf);
+
+ NavigableMap<byte[], byte[]> metricsResult = result
+ .getFamilyMap(EntityColumnFamily.METRICS.getInBytes());
+ checkMetricsSizeAndKey(metricsResult, metrics);
+ List<Cell> metricCells = result.getColumnCells(
+ EntityColumnFamily.METRICS.getInBytes(),
+ Bytes.toBytes(m1.getId()));
+ checkMetricsTimeseries(metricCells, m1);
+ }
+ }
+ assertEquals(1, rowCount);
+ assertEquals(15, colCount);
+
+ } finally {
+ hbi.stop();
+ hbi.close();
+ }
+ }
+
+ private void checkMetricsTimeseries(List<Cell> metricCells,
+ TimelineMetric m1) throws IOException {
+ Map<Long, Number> timeseries = m1.getValues();
+ assertEquals(metricCells.size(), timeseries.size());
+ for (Cell c1 : metricCells) {
+ assertTrue(timeseries.containsKey(c1.getTimestamp()));
+ assertEquals(GenericObjectMapper.read(CellUtil.cloneValue(c1)),
+ timeseries.get(c1.getTimestamp()));
+ }
+ }
+
+ private void checkMetricsSizeAndKey(
+ NavigableMap<byte[], byte[]> metricsResult, Set<TimelineMetric> metrics) {
+ assertEquals(metrics.size(), metricsResult.size());
+ for (TimelineMetric m1 : metrics) {
+ byte[] key = Bytes.toBytes(m1.getId());
+ assertTrue(metricsResult.containsKey(key));
+ }
+ }
+
+ private void checkConfigs(NavigableMap<byte[], byte[]> configValuesResult,
+ Map<String, String> conf) throws IOException {
+
+ assertEquals(conf.size(), configValuesResult.size());
+ byte[] columnName;
+ for (String key : conf.keySet()) {
+ columnName = Bytes.toBytes(key);
+ assertTrue(configValuesResult.containsKey(columnName));
+ byte[] value = configValuesResult.get(columnName);
+ assertNotNull(value);
+ assertEquals(conf.get(key), GenericObjectMapper.read(value));
+ }
+ }
+
+ private void checkRelatedEntities(Map<String, Set<String>> isRelatedTo,
+ NavigableMap<byte[], byte[]> infoValues, byte[] columnPrefix)
+ throws IOException {
+
+ for (String key : isRelatedTo.keySet()) {
+ byte[] columnName = TimelineWriterUtils.join(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES, columnPrefix,
+ Bytes.toBytes(key));
+
+ byte[] value = infoValues.get(columnName);
+ assertNotNull(value);
+ String isRelatedToEntities = GenericObjectMapper.read(value).toString();
+ assertNotNull(isRelatedToEntities);
+ assertEquals(
+ TimelineWriterUtils.getValueAsString(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR,
+ isRelatedTo.get(key)), isRelatedToEntities);
+ }
+ }
+
+ private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
+ String flow, Long runid, String appName, TimelineEntity te) {
+
+ byte[][] rowKeyComponents = TimelineWriterUtils.split(rowKey,
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES);
+
+ assertTrue(rowKeyComponents.length == 7);
+ assertEquals(user, Bytes.toString(rowKeyComponents[0]));
+ assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
+ assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
+ assertEquals(TimelineWriterUtils.encodeRunId(runid),
+ Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(TimelineWriterUtils.cleanse(appName), Bytes.toString(rowKeyComponents[4]));
+ assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
+ assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
+ return true;
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ util.shutdownMiniCluster();
+ }
+}
[07/50] [abbrv] hadoop git commit: YARN-3529. Added mini HBase
cluster and Phoenix support to timeline service v2 unit tests. Contributed by
Li Lu.
Posted by sj...@apache.org.
YARN-3529. Added mini HBase cluster and Phoenix support to timeline service v2 unit tests. Contributed by Li Lu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0478e1d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0478e1d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0478e1d1
Branch: refs/heads/YARN-2928-rebase
Commit: 0478e1d16477247304acb28c8bb55f6e3abe2484
Parents: d103fbe
Author: Zhijie Shen <zj...@apache.org>
Authored: Tue May 12 13:53:38 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 14:56:32 2015 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 51 +++++++++++++++
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop-yarn-server-timelineservice/pom.xml | 47 ++++++++++----
.../storage/PhoenixTimelineWriterImpl.java | 22 ++++++-
.../storage/TestPhoenixTimelineWriterImpl.java | 65 ++++++++++++++------
5 files changed, 154 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0478e1d1/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 611007f..389381b 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -46,6 +46,8 @@
<xerces.jdiff.version>2.11.0</xerces.jdiff.version>
<kafka.version>0.8.2.1</kafka.version>
+ <hbase.version>1.0.1</hbase.version>
+ <phoenix.version>4.5.0-SNAPSHOT</phoenix.version>
<hadoop.assemblies.version>${project.version}</hadoop.assemblies.version>
<commons-daemon.version>1.0.13</commons-daemon.version>
@@ -999,6 +1001,55 @@
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <version>${phoenix.version}</version>
+ <exclusions>
+ <!-- Exclude jline from here -->
+ <exclusion>
+ <artifactId>jline</artifactId>
+ <groupId>jline</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <type>test-jar</type>
+ <version>${phoenix.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.jruby</groupId>
+ <artifactId>jruby-complete</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0478e1d1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a05e4f6..ee21af8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -67,6 +67,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3134. Implemented Phoenix timeline writer to access HBase backend. (Li
Lu via zjshen)
+ YARN-3529. Added mini HBase cluster and Phoenix support to timeline service
+ v2 unit tests. (Li Lu via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0478e1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index f62230f..1e914de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -122,21 +122,44 @@
</dependency>
<dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-core</artifactId>
- <version>4.3.0</version>
- <exclusions>
- <!-- Exclude jline from here -->
- <exclusion>
- <artifactId>jline</artifactId>
- <groupId>jline</groupId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </dependency>
+ <!-- for unit tests only -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <scope>test</scope>
+ <optional>true</optional>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0478e1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
index af8a233..5b4442c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
@@ -43,6 +44,7 @@ import java.sql.Statement;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
@Private
@@ -50,6 +52,13 @@ import java.util.Set;
public class PhoenixTimelineWriterImpl extends AbstractService
implements TimelineWriter {
+ public static final String TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR
+ = YarnConfiguration.TIMELINE_SERVICE_PREFIX
+ + "writer.phoenix.connectionString";
+
+ public static final String TIMELINE_SERVICE_PHEONIX_STORAGE_CONN_STR_DEFAULT
+ = "jdbc:phoenix:localhost:2181:/hbase";
+
private static final Log LOG
= LogFactory.getLog(PhoenixTimelineWriterImpl.class);
private static final String PHOENIX_COL_FAMILY_PLACE_HOLDER
@@ -90,7 +99,10 @@ public class PhoenixTimelineWriterImpl extends AbstractService
private static final String PHOENIX_STORAGE_SEPARATOR = ";";
/** Connection string to the deployed Phoenix cluster */
- static final String CONN_STRING = "jdbc:phoenix:localhost:2181:/hbase";
+ @VisibleForTesting
+ String connString = null;
+ @VisibleForTesting
+ Properties connProperties = new Properties();
PhoenixTimelineWriterImpl() {
super((PhoenixTimelineWriterImpl.class.getName()));
@@ -98,6 +110,10 @@ public class PhoenixTimelineWriterImpl extends AbstractService
@Override
protected void serviceInit(Configuration conf) throws Exception {
+ // so check it here and only read in the config if it's not overridden.
+ connString =
+ conf.get(TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR,
+ TIMELINE_SERVICE_PHEONIX_STORAGE_CONN_STR_DEFAULT);
createTables();
super.init(conf);
}
@@ -174,11 +190,11 @@ public class PhoenixTimelineWriterImpl extends AbstractService
// Utility functions
@Private
@VisibleForTesting
- static Connection getConnection() throws IOException {
+ Connection getConnection() throws IOException {
Connection conn;
try {
Class.forName(DRIVER_CLASS_NAME);
- conn = DriverManager.getConnection(CONN_STRING);
+ conn = DriverManager.getConnection(connString, connProperties);
conn.setAutoCommit(false);
} catch (SQLException se) {
LOG.error("Failed to connect to phoenix server! "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0478e1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
index a55893e..dece83d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
@@ -23,30 +23,37 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
-public class TestPhoenixTimelineWriterImpl {
- private PhoenixTimelineWriterImpl writer;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
- @Before
- public void setup() throws Exception {
- // TODO: launch a miniphoenix cluster, or else we're directly operating on
- // the active Phoenix cluster
+public class TestPhoenixTimelineWriterImpl extends BaseTest {
+ private static PhoenixTimelineWriterImpl writer;
+ private static final int BATCH_SIZE = 3;
+
+ @BeforeClass
+ public static void setup() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- writer = createPhoenixWriter(conf);
+ writer = setupPhoenixClusterAndWriterForTest(conf);
}
- @Ignore
- @Test
+ @Test(timeout = 90000)
public void testPhoenixWriterBasic() throws Exception {
// Set up a list of timeline entities and write them back to Phoenix
int numEntity = 12;
@@ -91,28 +98,48 @@ public class TestPhoenixTimelineWriterImpl {
verifySQLWithCount(sql, (numEntity / 4), "Number of events should be ");
}
- @After
- public void cleanup() throws Exception {
- // Note: it is assumed that we're working on a test only cluster, or else
- // this cleanup process will drop the entity table.
+ @AfterClass
+ public static void cleanup() throws Exception {
writer.dropTable(PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME);
writer.dropTable(PhoenixTimelineWriterImpl.EVENT_TABLE_NAME);
writer.dropTable(PhoenixTimelineWriterImpl.METRIC_TABLE_NAME);
writer.serviceStop();
+ tearDownMiniCluster();
}
- private static PhoenixTimelineWriterImpl createPhoenixWriter(
+ private static PhoenixTimelineWriterImpl setupPhoenixClusterAndWriterForTest(
YarnConfiguration conf) throws Exception{
+ Map<String, String> props = new HashMap<>();
+ // Must update config before starting server
+ props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
+ Boolean.FALSE.toString());
+ props.put("java.security.krb5.realm", "");
+ props.put("java.security.krb5.kdc", "");
+ props.put(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER,
+ Boolean.FALSE.toString());
+ props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
+ props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
+ // Make a small batch size to test multiple calls to reserve sequences
+ props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB,
+ Long.toString(BATCH_SIZE));
+ // Must update config before starting server
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+
PhoenixTimelineWriterImpl myWriter = new PhoenixTimelineWriterImpl();
+ // Change connection settings for test
+ conf.set(
+ PhoenixTimelineWriterImpl.TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR,
+ getUrl());
+ myWriter.connProperties = PropertiesUtil.deepCopy(TEST_PROPERTIES);
myWriter.serviceInit(conf);
return myWriter;
}
private void verifySQLWithCount(String sql, int targetCount, String message)
- throws Exception{
+ throws Exception {
try (
Statement stmt =
- PhoenixTimelineWriterImpl.getConnection().createStatement();
+ writer.getConnection().createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
assertTrue("Result set empty on statement " + sql, rs.next());
assertNotNull("Fail to execute query " + sql, rs);
[40/50] [abbrv] hadoop git commit: YARN-3901. Populate flow run data
in the flow_run & flow activity tables (Vrushali C via sjlee)
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
new file mode 100644
index 0000000..af8df99
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+
+/**
+ * The flow activity table has column family info
+ * Stores the daily activity record for flows
+ * Useful as a quick lookup of what flows were
+ * running on a given day
+ *
+ * Example flow activity table record:
+ *
+ * </pre>
+ * |-------------------------------------------|
+ * | Row key | Column Family |
+ * | | info |
+ * |-------------------------------------------|
+ * | clusterId! | r!runid1:version1 |
+ * | inv Top of | |
+ * | Day! | r!runid2:version7 |
+ * | userName! | |
+ * | flowId | |
+ * |-------------------------------------------|
+ * </pre>
+ */
+public class FlowActivityTable extends BaseTable<FlowActivityTable> {
+ /** flow activity table prefix */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".flowactivity";
+
+ /** config param name that specifies the flowactivity table name */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /** default value for flowactivity table name */
+ public static final String DEFAULT_TABLE_NAME = "timelineservice.flowactivity";
+
+ private static final Log LOG = LogFactory.getLog(FlowActivityTable.class);
+
+ /** default max number of versions */
+ public static final int DEFAULT_METRICS_MAX_VERSIONS = Integer.MAX_VALUE;
+
+ public FlowActivityTable() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
+ * (org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor FlowActivityTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(FlowActivityColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ FlowActivityTableDescp.addFamily(infoCF);
+ infoCF.setMinVersions(1);
+ infoCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
+
+ // TODO: figure the split policy before running in production
+ admin.createTable(FlowActivityTableDescp);
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
new file mode 100644
index 0000000..ad30add
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies fully qualified columns for the {@link FlowRunTable}.
+ */
+public enum FlowRunColumn implements Column<FlowRunTable> {
+
+ /**
+ * When the flow was started. This is the minimum of currently known
+ * application start times.
+ */
+ MIN_START_TIME(FlowRunColumnFamily.INFO, "min_start_time",
+ AggregationOperation.MIN),
+
+ /**
+ * When the flow ended. This is the maximum of currently known application end
+ * times.
+ */
+ MAX_END_TIME(FlowRunColumnFamily.INFO, "max_end_time",
+ AggregationOperation.MAX),
+
+ /**
+ * The version of the flow that this flow belongs to.
+ */
+ FLOW_VERSION(FlowRunColumnFamily.INFO, "flow_version", null);
+
+ private final ColumnHelper<FlowRunTable> column;
+ private final ColumnFamily<FlowRunTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+ private final AggregationOperation aggOp;
+
+ private FlowRunColumn(ColumnFamily<FlowRunTable> columnFamily,
+ String columnQualifier, AggregationOperation aggOp) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ this.aggOp = aggOp;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes = Bytes.toBytes(Separator.SPACE
+ .encode(columnQualifier));
+ this.column = new ColumnHelper<FlowRunTable>(columnFamily);
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public byte[] getColumnQualifierBytes() {
+ return columnQualifierBytes.clone();
+ }
+
+ public AggregationOperation getAggregationOperation() {
+ return aggOp;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.Column#store
+ * (byte[], org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.Long, java.lang.Object,
+ * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[])
+ */
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<FlowRunTable> tableMutator, Long timestamp,
+ Object inputValue, Attribute... attributes) throws IOException {
+
+ Attribute[] combinedAttributes = TimelineWriterUtils.combineAttributes(
+ attributes, aggOp);
+ column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
+ inputValue, combinedAttributes);
+ }
+
+ public Object readResult(Result result) throws IOException {
+ return column.readResult(result, columnQualifierBytes);
+ }
+
+ /**
+ * Retrieve an {@link FlowRunColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(x) == columnFor(y)} if
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnQualifier
+ * Name of the column to retrieve
+ * @return the corresponding {@link FlowRunColumn} or null
+ */
+ public static final FlowRunColumn columnFor(String columnQualifier) {
+
+ // Match column based on value, assume column family matches.
+ for (FlowRunColumn ec : FlowRunColumn.values()) {
+ // Find a match based only on name.
+ if (ec.getColumnQualifier().equals(columnQualifier)) {
+ return ec;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link FlowRunColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)}
+ * if and only if {@code a.equals(b) & x.equals(y)} or
+ * {@code (x == y == null)}
+ *
+ * @param columnFamily
+ * The columnFamily for which to retrieve the column.
+ * @param name
+ * Name of the column to retrieve
+ * @return the corresponding {@link FlowRunColumn} or null if both arguments
+ * don't match.
+ */
+ public static final FlowRunColumn columnFor(FlowRunColumnFamily columnFamily,
+ String name) {
+
+ for (FlowRunColumn ec : FlowRunColumn.values()) {
+ // Find a match based column family and on name.
+ if (ec.columnFamily.equals(columnFamily)
+ && ec.getColumnQualifier().equals(name)) {
+ return ec;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java
new file mode 100644
index 0000000..8faf5f8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the flow run table column families.
+ */
+public enum FlowRunColumnFamily implements ColumnFamily<FlowRunTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value
+ * create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ private FlowRunColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
new file mode 100644
index 0000000..d55f510
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies partially qualified columns for the {@link FlowRunTable}.
+ */
+public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
+
+ /**
+ * To store flow run info values.
+ */
+ METRIC(FlowRunColumnFamily.INFO, "m", AggregationOperation.SUM);
+
+ private final ColumnHelper<FlowRunTable> column;
+ private final ColumnFamily<FlowRunTable> columnFamily;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+
+ private final AggregationOperation aggOp;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily
+ * that this column is stored in.
+ * @param columnPrefix
+ * for this column.
+ */
+ private FlowRunColumnPrefix(ColumnFamily<FlowRunTable> columnFamily,
+ String columnPrefix, AggregationOperation fra) {
+ column = new ColumnHelper<FlowRunTable>(columnFamily);
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes = Bytes.toBytes(Separator.SPACE
+ .encode(columnPrefix));
+ }
+ this.aggOp = fra;
+ }
+
+ /**
+ * @return the column name value
+ */
+ public String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ public byte[] getColumnPrefixBytes() {
+ return columnPrefixBytes.clone();
+ }
+
+ public AggregationOperation getAttribute() {
+ return aggOp;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ */
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<FlowRunTable> tableMutator, String qualifier,
+ Long timestamp, Object inputValue, Attribute... attributes)
+ throws IOException {
+
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier = ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifier);
+ Attribute[] combinedAttributes = TimelineWriterUtils.combineAttributes(
+ attributes, this.aggOp);
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
+ combinedAttributes);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ */
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<FlowRunTable> tableMutator, byte[] qualifier,
+ Long timestamp, Object inputValue, Attribute... attributes)
+ throws IOException {
+
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier = ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifier);
+ Attribute[] combinedAttributes = TimelineWriterUtils.combineAttributes(
+ attributes, this.aggOp);
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
+ combinedAttributes);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
+ */
+ public Object readResult(Result result, String qualifier) throws IOException {
+ byte[] columnQualifier = ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifier);
+ return column.readResult(result, columnQualifier);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResults(org.apache.hadoop.hbase.client.Result)
+ */
+ public Map<String, Object> readResults(Result result) throws IOException {
+ return column.readResults(result, columnPrefixBytes);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
+ */
+ public <T> NavigableMap<String, NavigableMap<Long, T>> readResultsWithTimestamps(
+ Result result) throws IOException {
+ return column.readResultsWithTimestamps(result, columnPrefixBytes);
+ }
+
+ /**
+ * Retrieve an {@link FlowRunColumnPrefix} given a name, or null if there is
+ * no match. The following holds true: {@code columnFor(x) == columnFor(y)} if
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnPrefix
+ * Name of the column to retrieve
+ * @return the corresponding {@link FlowRunColumnPrefix} or null
+ */
+ public static final FlowRunColumnPrefix columnFor(String columnPrefix) {
+
+ // Match column based on value, assume column family matches.
+ for (FlowRunColumnPrefix frcp : FlowRunColumnPrefix.values()) {
+ // Find a match based only on name.
+ if (frcp.getColumnPrefix().equals(columnPrefix)) {
+ return frcp;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link FlowRunColumnPrefix} given a name, or null if there is
+ * no match. The following holds true:
+ * {@code columnFor(a,x) == columnFor(b,y)} if and only if
+ * {@code (x == y == null)} or {@code a.equals(b) & x.equals(y)}
+ *
+ * @param columnFamily
+ * The columnFamily for which to retrieve the column.
+ * @param columnPrefix
+ * Name of the column to retrieve
+ * @return the corresponding {@link FlowRunColumnPrefix} or null if both
+ * arguments don't match.
+ */
+ public static final FlowRunColumnPrefix columnFor(
+ FlowRunColumnFamily columnFamily, String columnPrefix) {
+
+ // TODO: needs unit test to confirm and need to update javadoc to explain
+ // null prefix case.
+
+ for (FlowRunColumnPrefix frcp : FlowRunColumnPrefix.values()) {
+ // Find a match based column family and on name.
+ if (frcp.columnFamily.equals(columnFamily)
+ && (((columnPrefix == null) && (frcp.getColumnPrefix() == null)) || (frcp
+ .getColumnPrefix().equals(columnPrefix)))) {
+ return frcp;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
new file mode 100644
index 0000000..f743e5e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
+
+public class FlowRunCoprocessor extends BaseRegionObserver {
+
+ @SuppressWarnings("unused")
+ private static final Log LOG = LogFactory.getLog(FlowRunCoprocessor.class);
+
+ private HRegion region;
+ /**
+ * generate a timestamp that is unique per row in a region this is per region
+ */
+ private final TimestampGenerator timestampGenerator = new TimestampGenerator();
+
+ @Override
+ public void start(CoprocessorEnvironment e) throws IOException {
+ if (e instanceof RegionCoprocessorEnvironment) {
+ RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
+ this.region = env.getRegion();
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * This method adds the tags onto the cells in the Put. It is presumed that
+ * all the cells in one Put have the same set of Tags. The existing cell
+ * timestamp is overwritten for non-metric cells and each such cell gets a new
+ * unique timestamp generated by {@link TimestampGenerator}
+ *
+ * @see
+ * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#prePut(org.apache
+ * .hadoop.hbase.coprocessor.ObserverContext,
+ * org.apache.hadoop.hbase.client.Put,
+ * org.apache.hadoop.hbase.regionserver.wal.WALEdit,
+ * org.apache.hadoop.hbase.client.Durability)
+ */
+ @Override
+ public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put put,
+ WALEdit edit, Durability durability) throws IOException {
+ Map<String, byte[]> attributes = put.getAttributesMap();
+
+ // Assumption is that all the cells in a put are the same operation.
+ List<Tag> tags = new ArrayList<>();
+ if ((attributes != null) && (attributes.size() > 0)) {
+ for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
+ Tag t = TimelineWriterUtils.getTagFromAttribute(attribute);
+ tags.add(t);
+ }
+ byte[] tagByteArray = Tag.fromList(tags);
+ NavigableMap<byte[], List<Cell>> newFamilyMap = new TreeMap<>(
+ Bytes.BYTES_COMPARATOR);
+ for (Map.Entry<byte[], List<Cell>> entry : put.getFamilyCellMap()
+ .entrySet()) {
+ List<Cell> newCells = new ArrayList<>(entry.getValue().size());
+ for (Cell cell : entry.getValue()) {
+ // for each cell in the put add the tags
+ // Assumption is that all the cells in
+ // one put are the same operation
+ // also, get a unique cell timestamp for non-metric cells
+ // this way we don't inadvertently overwrite cell versions
+ long cellTimestamp = getCellTimestamp(cell.getTimestamp(), tags);
+ newCells.add(CellUtil.createCell(CellUtil.cloneRow(cell),
+ CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
+ cellTimestamp, KeyValue.Type.Put, CellUtil.cloneValue(cell),
+ tagByteArray));
+ }
+ newFamilyMap.put(entry.getKey(), newCells);
+ } // for each entry
+ // Update the family map for the Put
+ put.setFamilyCellMap(newFamilyMap);
+ }
+ }
+
+ /**
+ * Determines if the current cell's timestamp is to be used or a new unique
+ * cell timestamp is to be used. The reason this is done is to inadvertently
+ * overwrite cells when writes come in very fast. But for metric cells, the
+ * cell timestamp signifies the metric timestamp. Hence we don't want to
+ * overwrite it.
+ *
+ * @param timestamp
+ * @param tags
+ * @return cell timestamp
+ */
+ private long getCellTimestamp(long timestamp, List<Tag> tags) {
+ // if ts not set (hbase sets to HConstants.LATEST_TIMESTAMP by default)
+ // then use the generator
+ if (timestamp == HConstants.LATEST_TIMESTAMP) {
+ return timestampGenerator.getUniqueTimestamp();
+ } else {
+ return timestamp;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Creates a {@link FlowScanner} Scan so that it can correctly process the
+ * contents of {@link FlowRunTable}.
+ *
+ * @see
+ * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#preGetOp(org.apache
+ * .hadoop.hbase.coprocessor.ObserverContext,
+ * org.apache.hadoop.hbase.client.Get, java.util.List)
+ */
+ @Override
+ public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e,
+ Get get, List<Cell> results) throws IOException {
+ Scan scan = new Scan(get);
+ scan.setMaxVersions();
+ RegionScanner scanner = null;
+ try {
+ scanner = new FlowScanner(region, scan.getBatch(),
+ region.getScanner(scan));
+ scanner.next(results);
+ e.bypass();
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Ensures that max versions are set for the Scan so that metrics can be
+ * correctly aggregated and min/max can be correctly determined.
+ *
+ * @see
+ * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#preScannerOpen(org
+ * .apache.hadoop.hbase.coprocessor.ObserverContext,
+ * org.apache.hadoop.hbase.client.Scan,
+ * org.apache.hadoop.hbase.regionserver.RegionScanner)
+ */
+ @Override
+ public RegionScanner preScannerOpen(
+ ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
+ RegionScanner s) throws IOException {
+ // set max versions for scan to see all
+ // versions to aggregate for metrics
+ scan.setMaxVersions();
+ return s;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Creates a {@link FlowScanner} Scan so that it can correctly process the
+ * contents of {@link FlowRunTable}.
+ *
+ * @see
+ * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#postScannerOpen(
+ * org.apache.hadoop.hbase.coprocessor.ObserverContext,
+ * org.apache.hadoop.hbase.client.Scan,
+ * org.apache.hadoop.hbase.regionserver.RegionScanner)
+ */
+ @Override
+ public RegionScanner postScannerOpen(
+ ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
+ RegionScanner scanner) throws IOException {
+ return new FlowScanner(region, scan.getBatch(), scanner);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
new file mode 100644
index 0000000..e133241
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+
+/**
+ * Represents a rowkey for the flow run table.
+ */
+public class FlowRunRowKey {
+ // TODO: more methods are needed for this class like parse row key
+
+ /**
+ * Constructs a row key for the entity table as follows: {
+ * clusterId!userI!flowId!Inverted Flow Run Id}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @return byte array with the row key
+ */
+ public static byte[] getRowKey(String clusterId, String userId,
+ String flowId, Long flowRunId) {
+ byte[] first = Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId,
+ userId, flowId));
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
+ return Separator.QUALIFIERS.join(first, second);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
new file mode 100644
index 0000000..b1b93c1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+
+/**
+ * The flow run table has column family info
+ * Stores per flow run information
+ * aggregated across applications.
+ *
+ * Metrics are also stored in the info column family.
+ *
+ * Example flow run table record:
+ *
+ * <pre>
+ * flow_run table
+ * |-------------------------------------------|
+ * | Row key | Column Family |
+ * | | info |
+ * |-------------------------------------------|
+ * | clusterId! | flow_version:version7 |
+ * | userName! | |
+ * | flowId! | running_apps:1 |
+ * | flowRunId | |
+ * | | min_start_time:1392995080000 |
+ * | | #0:"" |
+ * | | |
+ * | | min_start_time:1392995081012 |
+ * | | #0:appId2 |
+ * | | |
+ * | | min_start_time:1392993083210 |
+ * | | #0:appId3 |
+ * | | |
+ * | | |
+ * | | max_end_time:1392993084018 |
+ * | | #0:"" |
+ * | | |
+ * | | |
+ * | | m!mapInputRecords:127 |
+ * | | #0:"" |
+ * | | |
+ * | | m!mapInputRecords:31 |
+ * | | #2:appId2 |
+ * | | |
+ * | | m!mapInputRecords:37 |
+ * | | #1:appId3 |
+ * | | |
+ * | | |
+ * | | m!mapOutputRecords:181 |
+ * | | #0:"" |
+ * | | |
+ * | | m!mapOutputRecords:37 |
+ * | | #1:appId3 |
+ * | | |
+ * | | |
+ * |-------------------------------------------|
+ * </pre>
+ */
+public class FlowRunTable extends BaseTable<FlowRunTable> {
+ /** entity prefix */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".flowrun";
+
+ /** config param name that specifies the flowrun table name */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /** default value for flowrun table name */
+ public static final String DEFAULT_TABLE_NAME = "timelineservice.flowrun";
+
+ private static final Log LOG = LogFactory.getLog(FlowRunTable.class);
+
+ /** default max number of versions */
+ public static final int DEFAULT_METRICS_MAX_VERSIONS = Integer.MAX_VALUE;
+
+ public FlowRunTable() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
+ * (org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor flowRunTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(FlowRunColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ flowRunTableDescp.addFamily(infoCF);
+ infoCF.setMinVersions(1);
+ infoCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
+
+ // TODO: figure the split policy
+ flowRunTableDescp.addCoprocessor(FlowRunCoprocessor.class
+ .getCanonicalName());
+ admin.createTable(flowRunTableDescp);
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
new file mode 100644
index 0000000..a1948aa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
@@ -0,0 +1,486 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+/**
+ * Invoked via the coprocessor when a Get or a Scan is issued for flow run
+ * table. Looks through the list of cells per row, checks their tags and does
+ * operation on those cells as per the cell tags. Transforms reads of the stored
+ * metrics into calculated sums for each column Also, finds the min and max for
+ * start and end times in a flow run.
+ */
+class FlowScanner implements RegionScanner, Closeable {
+
+ private static final Log LOG = LogFactory.getLog(FlowScanner.class);
+
+ private final HRegion region;
+ private final InternalScanner flowRunScanner;
+ private RegionScanner regionScanner;
+ private final int limit;
+ private boolean hasMore;
+ private byte[] currentRow;
+ private List<Cell> availableCells = new ArrayList<>();
+ private int currentIndex;
+
+ FlowScanner(HRegion region, int limit, InternalScanner internalScanner) {
+ this.region = region;
+ this.limit = limit;
+ this.flowRunScanner = internalScanner;
+ if (internalScanner instanceof RegionScanner) {
+ this.regionScanner = (RegionScanner) internalScanner;
+ }
+ // TODO: note if it's compaction/flush
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.hbase.regionserver.RegionScanner#getRegionInfo()
+ */
+ @Override
+ public HRegionInfo getRegionInfo() {
+ return region.getRegionInfo();
+ }
+
+ @Override
+ public boolean nextRaw(List<Cell> cells) throws IOException {
+ return nextRaw(cells, limit);
+ }
+
+ @Override
+ public boolean nextRaw(List<Cell> cells, int limit) throws IOException {
+ return nextInternal(cells, limit);
+ }
+
+ @Override
+ public boolean next(List<Cell> cells) throws IOException {
+ return next(cells, limit);
+ }
+
+ @Override
+ public boolean next(List<Cell> cells, int limit) throws IOException {
+ return nextInternal(cells, limit);
+ }
+
+ private String getAggregationCompactionDimension(List<Tag> tags) {
+ String appId = null;
+ for (Tag t : tags) {
+ if (AggregationCompactionDimension.APPLICATION_ID.getTagType() == t
+ .getType()) {
+ appId = Bytes.toString(t.getValue());
+ }
+ }
+ return appId;
+ }
+
+ /**
+ * This method loops through the cells in a given row of the
+ * {@link FlowRunTable}. It looks at the tags of each cell to figure out how
+ * to process the contents. It then calculates the sum or min or max for each
+ * column or returns the cell as is.
+ *
+ * @param cells
+ * @param limit
+ * @return true if next row is available for the scanner, false otherwise
+ * @throws IOException
+ */
+ private boolean nextInternal(List<Cell> cells, int limit) throws IOException {
+ Cell cell = null;
+ startNext();
+ // Loop through all the cells in this row
+ // For min/max/metrics we do need to scan the entire set of cells to get the
+ // right one
+ // But with flush/compaction, the number of cells being scanned will go down
+ // cells are grouped per column qualifier then sorted by cell timestamp
+ // (latest to oldest) per column qualifier
+ // So all cells in one qualifier come one after the other before we see the
+ // next column qualifier
+ ByteArrayComparator comp = new ByteArrayComparator();
+ byte[] currentColumnQualifier = TimelineWriterUtils.EMPTY_BYTES;
+ AggregationOperation currentAggOp = null;
+ SortedSet<Cell> currentColumnCells = new TreeSet<>(KeyValue.COMPARATOR);
+ Set<String> alreadySeenAggDim = new HashSet<>();
+ int addedCnt = 0;
+ while (((cell = peekAtNextCell(limit)) != null)
+ && (limit <= 0 || addedCnt < limit)) {
+ byte[] newColumnQualifier = CellUtil.cloneQualifier(cell);
+ if (comp.compare(currentColumnQualifier, newColumnQualifier) != 0) {
+ addedCnt += emitCells(cells, currentColumnCells, currentAggOp);
+ resetState(currentColumnCells, alreadySeenAggDim);
+ currentColumnQualifier = newColumnQualifier;
+ currentAggOp = getCurrentAggOp(cell);
+ }
+ collectCells(currentColumnCells, currentAggOp, cell, alreadySeenAggDim);
+ nextCell(limit);
+ }
+ if (!currentColumnCells.isEmpty()) {
+ emitCells(cells, currentColumnCells, currentAggOp);
+ }
+ return hasMore();
+ }
+
+ private AggregationOperation getCurrentAggOp(Cell cell) {
+ List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
+ cell.getTagsLength());
+ // We assume that all the operations for a particular column are the same
+ return TimelineWriterUtils.getAggregationOperationFromTagsList(tags);
+ }
+
+ /**
+ * resets the parameters to an intialized state for next loop iteration
+ *
+ * @param cell
+ * @param currentAggOp
+ * @param currentColumnCells
+ * @param alreadySeenAggDim
+ * @param collectedButNotEmitted
+ */
+ private void resetState(SortedSet<Cell> currentColumnCells,
+ Set<String> alreadySeenAggDim) {
+ currentColumnCells.clear();
+ alreadySeenAggDim.clear();
+ }
+
+ private void collectCells(SortedSet<Cell> currentColumnCells,
+ AggregationOperation currentAggOp, Cell cell,
+ Set<String> alreadySeenAggDim) throws IOException {
+ if (currentAggOp == null) {
+ // not a min/max/metric cell, so just return it as is
+ currentColumnCells.add(cell);
+ nextCell(limit);
+ return;
+ }
+
+ switch (currentAggOp) {
+ case MIN:
+ if (currentColumnCells.size() == 0) {
+ currentColumnCells.add(cell);
+ } else {
+ Cell currentMinCell = currentColumnCells.first();
+ Cell newMinCell = compareCellValues(currentMinCell, cell, currentAggOp);
+ if (!currentMinCell.equals(newMinCell)) {
+ currentColumnCells.remove(currentMinCell);
+ currentColumnCells.add(newMinCell);
+ }
+ }
+ break;
+ case MAX:
+ if (currentColumnCells.size() == 0) {
+ currentColumnCells.add(cell);
+ } else {
+ Cell currentMaxCell = currentColumnCells.first();
+ Cell newMaxCell = compareCellValues(currentMaxCell, cell, currentAggOp);
+ if (!currentMaxCell.equals(newMaxCell)) {
+ currentColumnCells.remove(currentMaxCell);
+ currentColumnCells.add(newMaxCell);
+ }
+ }
+ break;
+ case SUM:
+ case SUM_FINAL:
+ // only if this app has not been seen yet, add to current column cells
+ List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
+ cell.getTagsLength());
+ String aggDim = getAggregationCompactionDimension(tags);
+ if (alreadySeenAggDim.contains(aggDim)) {
+ // if this agg dimension has already been seen,
+ // since they show up in sorted order
+ // we drop the rest which are older
+ // in other words, this cell is older than previously seen cells
+ // for that agg dim
+ } else {
+ // not seen this agg dim, hence consider this cell in our working set
+ currentColumnCells.add(cell);
+ alreadySeenAggDim.add(aggDim);
+ }
+ break;
+ default:
+ break;
+ } // end of switch case
+ }
+
+ /*
+ * Processes the cells in input param currentColumnCells and populates
+ * List<Cell> cells as the output based on the input AggregationOperation
+ * parameter.
+ */
+ private int emitCells(List<Cell> cells, SortedSet<Cell> currentColumnCells,
+ AggregationOperation currentAggOp) throws IOException {
+ if ((currentColumnCells == null) || (currentColumnCells.size() == 0)) {
+ return 0;
+ }
+ if (currentAggOp == null) {
+ cells.addAll(currentColumnCells);
+ return currentColumnCells.size();
+ }
+
+ switch (currentAggOp) {
+ case MIN:
+ case MAX:
+ cells.addAll(currentColumnCells);
+ return currentColumnCells.size();
+ case SUM:
+ case SUM_FINAL:
+ Cell sumCell = processSummation(currentColumnCells);
+ cells.add(sumCell);
+ return 1;
+ default:
+ cells.addAll(currentColumnCells);
+ return currentColumnCells.size();
+ }
+ }
+
+ /*
+ * Returns a cell whose value is the sum of all cell values in the input set.
+ * The new cell created has the timestamp of the most recent metric cell. The
+ * sum of a metric for a flow run is the summation at the point of the last
+ * metric update in that flow till that time.
+ */
+ private Cell processSummation(SortedSet<Cell> currentColumnCells)
+ throws IOException {
+ Number sum = 0;
+ Number currentValue = 0;
+ long ts = 0L;
+ long mostCurrentTimestamp = 0l;
+ Cell mostRecentCell = null;
+ for (Cell cell : currentColumnCells) {
+ currentValue = (Number) GenericObjectMapper.read(CellUtil
+ .cloneValue(cell));
+ ts = cell.getTimestamp();
+ if (mostCurrentTimestamp < ts) {
+ mostCurrentTimestamp = ts;
+ mostRecentCell = cell;
+ }
+ sum = sum.longValue() + currentValue.longValue();
+ }
+ Cell sumCell = createNewCell(mostRecentCell, sum);
+ return sumCell;
+ }
+
+ /**
+ * Determines which cell is to be returned based on the values in each cell
+ * and the comparison operation MIN or MAX.
+ *
+ * @param previouslyChosenCell
+ * @param currentCell
+ * @param currentAggOp
+ * @return the cell which is the min (or max) cell
+ * @throws IOException
+ */
+ private Cell compareCellValues(Cell previouslyChosenCell, Cell currentCell,
+ AggregationOperation currentAggOp) throws IOException {
+ if (previouslyChosenCell == null) {
+ return currentCell;
+ }
+ try {
+ long previouslyChosenCellValue = ((Number) GenericObjectMapper
+ .read(CellUtil.cloneValue(previouslyChosenCell))).longValue();
+ long currentCellValue = ((Number) GenericObjectMapper.read(CellUtil
+ .cloneValue(currentCell))).longValue();
+ switch (currentAggOp) {
+ case MIN:
+ if (currentCellValue < previouslyChosenCellValue) {
+ // new value is minimum, hence return this cell
+ return currentCell;
+ } else {
+ // previously chosen value is miniumum, hence return previous min cell
+ return previouslyChosenCell;
+ }
+ case MAX:
+ if (currentCellValue > previouslyChosenCellValue) {
+ // new value is max, hence return this cell
+ return currentCell;
+ } else {
+ // previously chosen value is max, hence return previous max cell
+ return previouslyChosenCell;
+ }
+ default:
+ return currentCell;
+ }
+ } catch (IllegalArgumentException iae) {
+ LOG.error("caught iae during conversion to long ", iae);
+ return currentCell;
+ }
+ }
+
+ private Cell createNewCell(Cell origCell, Number number) throws IOException {
+ byte[] newValue = GenericObjectMapper.write(number);
+ return CellUtil.createCell(CellUtil.cloneRow(origCell),
+ CellUtil.cloneFamily(origCell), CellUtil.cloneQualifier(origCell),
+ origCell.getTimestamp(), KeyValue.Type.Put.getCode(), newValue);
+ }
+
+ @Override
+ public void close() throws IOException {
+ flowRunScanner.close();
+ }
+
+ /**
+ * Called to signal the start of the next() call by the scanner.
+ */
+ public void startNext() {
+ currentRow = null;
+ }
+
+ /**
+ * Returns whether or not the underlying scanner has more rows.
+ */
+ public boolean hasMore() {
+ return currentIndex < availableCells.size() ? true : hasMore;
+ }
+
+ /**
+ * Returns the next available cell for the current row and advances the
+ * pointer to the next cell. This method can be called multiple times in a row
+ * to advance through all the available cells.
+ *
+ * @param limit
+ * the limit of number of cells to return if the next batch must be
+ * fetched by the wrapped scanner
+ * @return the next available cell or null if no more cells are available for
+ * the current row
+ * @throws IOException
+ */
+ public Cell nextCell(int limit) throws IOException {
+ Cell cell = peekAtNextCell(limit);
+ if (cell != null) {
+ currentIndex++;
+ }
+ return cell;
+ }
+
+ /**
+ * Returns the next available cell for the current row, without advancing the
+ * pointer. Calling this method multiple times in a row will continue to
+ * return the same cell.
+ *
+ * @param limit
+ * the limit of number of cells to return if the next batch must be
+ * fetched by the wrapped scanner
+ * @return the next available cell or null if no more cells are available for
+ * the current row
+ * @throws IOException
+ */
+ public Cell peekAtNextCell(int limit) throws IOException {
+ if (currentIndex >= availableCells.size()) {
+ // done with current batch
+ availableCells.clear();
+ currentIndex = 0;
+ hasMore = flowRunScanner.next(availableCells, limit);
+ }
+ Cell cell = null;
+ if (currentIndex < availableCells.size()) {
+ cell = availableCells.get(currentIndex);
+ if (currentRow == null) {
+ currentRow = CellUtil.cloneRow(cell);
+ } else if (!CellUtil.matchingRow(cell, currentRow)) {
+ // moved on to the next row
+ // don't use the current cell
+ // also signal no more cells for this row
+ return null;
+ }
+ }
+ return cell;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.hbase.regionserver.RegionScanner#getMaxResultSize()
+ */
+ @Override
+ public long getMaxResultSize() {
+ if (regionScanner == null) {
+ throw new IllegalStateException(
+ "RegionScanner.isFilterDone() called when the flow "
+ + "scanner's scanner is not a RegionScanner");
+ }
+ return regionScanner.getMaxResultSize();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.hbase.regionserver.RegionScanner#getMvccReadPoint()
+ */
+ @Override
+ public long getMvccReadPoint() {
+ if (regionScanner == null) {
+ throw new IllegalStateException(
+ "RegionScanner.isFilterDone() called when the flow "
+ + "scanner's internal scanner is not a RegionScanner");
+ }
+ return regionScanner.getMvccReadPoint();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.hbase.regionserver.RegionScanner#isFilterDone()
+ */
+ @Override
+ public boolean isFilterDone() throws IOException {
+ if (regionScanner == null) {
+ throw new IllegalStateException(
+ "RegionScanner.isFilterDone() called when the flow "
+ + "scanner's internal scanner is not a RegionScanner");
+ }
+ return regionScanner.isFilterDone();
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.hbase.regionserver.RegionScanner#reseek(byte[])
+ */
+ @Override
+ public boolean reseek(byte[] bytes) throws IOException {
+ if (regionScanner == null) {
+ throw new IllegalStateException(
+ "RegionScanner.reseek() called when the flow "
+ + "scanner's internal scanner is not a RegionScanner");
+ }
+ return regionScanner.reseek(bytes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index 2875e01..3962341 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.application.Applica
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
@@ -88,20 +87,15 @@ public class TestHBaseTimelineStorage {
}
private static void createSchema() throws IOException {
- new EntityTable()
- .createTable(util.getHBaseAdmin(), util.getConfiguration());
- new AppToFlowTable()
- .createTable(util.getHBaseAdmin(), util.getConfiguration());
- new ApplicationTable()
- .createTable(util.getHBaseAdmin(), util.getConfiguration());
+ TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
}
@Test
public void testWriteApplicationToHBase() throws Exception {
TimelineEntities te = new TimelineEntities();
ApplicationEntity entity = new ApplicationEntity();
- String id = "hello";
- entity.setId(id);
+ String appId = "application_1000178881110_2002";
+ entity.setId(appId);
long cTime = 1425016501000L;
long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
@@ -173,12 +167,12 @@ public class TestHBaseTimelineStorage {
String flow = "some_flow_name";
String flowVersion = "AB7822C10F1111";
long runid = 1002345678919L;
- hbi.write(cluster, user, flow, flowVersion, runid, id, te);
+ hbi.write(cluster, user, flow, flowVersion, runid, appId, te);
hbi.stop();
// retrieve the row
byte[] rowKey =
- ApplicationRowKey.getRowKey(cluster, user, flow, runid, id);
+ ApplicationRowKey.getRowKey(cluster, user, flow, runid, appId);
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
Connection conn = ConnectionFactory.createConnection(c1);
@@ -190,11 +184,11 @@ public class TestHBaseTimelineStorage {
// check the row key
byte[] row1 = result.getRow();
assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
- id));
+ appId));
// check info column family
String id1 = ApplicationColumn.ID.readResult(result).toString();
- assertEquals(id, id1);
+ assertEquals(appId, id1);
Number val =
(Number) ApplicationColumn.CREATED_TIME.readResult(result);
@@ -252,17 +246,17 @@ public class TestHBaseTimelineStorage {
assertEquals(metricValues, metricMap);
// read the timeline entity using the reader this time
- TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, id,
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appId,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- id, entity.getType(), null, null, null, null, null, null, null,
+ appId, entity.getType(), null, null, null, null, null, null, null,
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
assertEquals(1, es1.size());
// verify attributes
- assertEquals(id, e1.getId());
+ assertEquals(appId, e1.getId());
assertEquals(TimelineEntityType.YARN_APPLICATION.toString(),
e1.getType());
assertEquals(cTime, e1.getCreatedTime());
@@ -576,7 +570,7 @@ public class TestHBaseTimelineStorage {
String flow = "other_flow_name";
String flowVersion = "1111F01C2287BA";
long runid = 1009876543218L;
- String appName = "some app name";
+ String appName = "application_123465899910_1001";
hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
hbi.stop();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e76c7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
new file mode 100644
index 0000000..f8331fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+
+/**
+ * Generates the data/entities for the FlowRun and FlowActivity Tables
+ */
+class TestFlowDataGenerator {
+
+ private final static String metric1 = "MAP_SLOT_MILLIS";
+ private final static String metric2 = "HDFS_BYTES_READ";
+
+
+ static TimelineEntity getEntityMetricsApp1() {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowRunMetrics_test";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ Long cTime = 1425016501000L;
+ entity.setCreatedTime(cTime);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId(metric1);
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 100000, 2);
+ metricValues.put(ts - 80000, 40);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+
+ TimelineMetric m2 = new TimelineMetric();
+ m2.setId(metric2);
+ metricValues = new HashMap<Long, Number>();
+ ts = System.currentTimeMillis();
+ metricValues.put(ts - 100000, 31);
+ metricValues.put(ts - 80000, 57);
+ m2.setType(Type.TIME_SERIES);
+ m2.setValues(metricValues);
+ metrics.add(m2);
+
+ entity.addMetrics(metrics);
+ return entity;
+ }
+
+ static TimelineEntity getEntityMetricsApp2() {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowRunMetrics_test";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ Long cTime = 1425016501000L;
+ entity.setCreatedTime(cTime);
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId(metric1);
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 100000, 5L);
+ metricValues.put(ts - 80000, 101L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+ return entity;
+ }
+
+ static TimelineEntity getEntity1() {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowRunHello";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ Long cTime = 20000000000000L;
+ Long mTime = 1425026901000L;
+ entity.setCreatedTime(cTime);
+ entity.setModifiedTime(mTime);
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId(metric1);
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 120000, 100000000);
+ metricValues.put(ts - 100000, 200000000);
+ metricValues.put(ts - 80000, 300000000);
+ metricValues.put(ts - 60000, 400000000);
+ metricValues.put(ts - 40000, 50000000000L);
+ metricValues.put(ts - 20000, 60000000000L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ Long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+ String expKey = "foo_event";
+ Object expVal = "test";
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
+
+ event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ event.setTimestamp(1436512801000L);
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
+
+ return entity;
+ }
+
+ static TimelineEntity getEntityGreaterStartTime() {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setCreatedTime(30000000000000L);
+ entity.setId("flowRunHello with greater start time");
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setType(type);
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ long endTs = 1439379885000L;
+ event.setTimestamp(endTs);
+ String expKey = "foo_event_greater";
+ String expVal = "test_app_greater";
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
+ return entity;
+ }
+
+ static TimelineEntity getEntityMaxEndTime(long endTs) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId("flowRunHello Max End time");
+ entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ event.setTimestamp(endTs);
+ String expKey = "foo_even_max_ finished";
+ String expVal = "test_app_max_finished";
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
+ return entity;
+ }
+
+ static TimelineEntity getEntityMinStartTime() {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowRunHelloMInStartTime";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ Long cTime = 10000000000000L;
+ entity.setCreatedTime(cTime);
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ event.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(event);
+ return entity;
+ }
+
+
+ static TimelineEntity getFlowApp1() {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowActivity_test";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ Long cTime = 1425016501000L;
+ entity.setCreatedTime(cTime);
+
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ Long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+ String expKey = "foo_event";
+ Object expVal = "test";
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
+
+ return entity;
+ }
+
+}
[50/50] [abbrv] hadoop git commit: Applied a slightly updated version
of patch v.2 for HDFS-9187 to resolve the regression on the trunk caused by
HDFS-9080.
Posted by sj...@apache.org.
Applied a slightly updated version of patch v.2 for HDFS-9187 to resolve the regression on the trunk caused by HDFS-9080.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd5af9c5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd5af9c5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd5af9c5
Branch: refs/heads/YARN-2928-rebase
Commit: bd5af9c5f4e3775042b9471aa54796d14d396be2
Parents: 6fec6f6
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Oct 12 09:48:42 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Mon Oct 12 09:48:42 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 7 -------
.../main/java/org/apache/hadoop/fs/Globber.java | 2 +-
.../org/apache/hadoop/tracing/TestTraceUtils.java | 18 ++++++++++++++++++
3 files changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd5af9c5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 6f13a87..bd8aa2a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -129,12 +129,6 @@ public abstract class FileSystem extends Configured implements Closeable {
boolean resolveSymlinks;
- private Tracer tracer;
-
- protected final Tracer getTracer() {
- return tracer;
- }
-
/**
* This method adds a file system for testing so that we can find it later. It
* is only for testing.
@@ -2712,7 +2706,6 @@ public abstract class FileSystem extends Configured implements Closeable {
try {
Class<?> clazz = getFileSystemClass(uri.getScheme(), conf);
FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
- fs.tracer = tracer;
fs.initialize(uri, conf);
return fs;
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd5af9c5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index 7a01575..7c69167 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -47,7 +47,7 @@ class Globber {
this.fc = null;
this.pathPattern = pathPattern;
this.filter = filter;
- this.tracer = fs.getTracer();
+ this.tracer = FsTracer.get(fs.getConf());
}
public Globber(FileContext fc, Path pathPattern, PathFilter filter) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd5af9c5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/TestTraceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/TestTraceUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/TestTraceUtils.java
index 75411da..fc0726e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/TestTraceUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/TestTraceUtils.java
@@ -18,8 +18,13 @@
package org.apache.hadoop.tracing;
import static org.junit.Assert.assertEquals;
+
+import java.net.URI;
import java.util.LinkedList;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.tracing.SpanReceiverInfo.ConfigurationPair;
import org.apache.htrace.core.HTraceConfiguration;
import org.junit.Test;
@@ -50,4 +55,17 @@ public class TestTraceUtils {
HTraceConfiguration wrapped = TraceUtils.wrapHadoopConf(TEST_PREFIX, conf, extraConfig);
assertEquals(newValue, wrapped.get(key));
}
+
+ /**
+ * Test tracing the globber. This is a regression test for HDFS-9187.
+ */
+ @Test
+ public void testTracingGlobber() throws Exception {
+ // Bypass the normal FileSystem object creation path by just creating an
+ // instance of a subclass.
+ FileSystem fs = new LocalFileSystem();
+ fs.initialize(new URI("file:///"), new Configuration());
+ fs.globStatus(new Path("/"));
+ fs.close();
+ }
}
[38/50] [abbrv] hadoop git commit: YARN-4102. Add a "skip existing
table" mode for timeline schema creator (Li Lu via sjlee)
Posted by sj...@apache.org.
YARN-4102. Add a "skip existing table" mode for timeline schema creator (Li Lu via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23fa8fa2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23fa8fa2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23fa8fa2
Branch: refs/heads/YARN-2928-rebase
Commit: 23fa8fa289f181131398f8875b79bd7b6aff5d5e
Parents: c51eceb
Author: Sangjin Lee <sj...@apache.org>
Authored: Fri Sep 11 09:46:13 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 17:05:03 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../storage/TimelineSchemaCreator.java | 81 ++++++++++++++++----
2 files changed, 69 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23fa8fa2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 43f615a..302b325 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -100,6 +100,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3814. REST API implementation for getting raw entities in
TimelineReader (Varun Saxena via sjlee)
+ YARN-4102. Add a "skip existing table" mode for timeline schema creator (Li
+ Lu via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23fa8fa2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index 5120856..e7e51a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -54,6 +54,11 @@ public class TimelineSchemaCreator {
final static String NAME = TimelineSchemaCreator.class.getSimpleName();
private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
private static final String PHOENIX_OPTION_SHORT = "p";
+ private static final String SKIP_EXISTING_TABLE_OPTION_SHORT = "s";
+ private static final String APP_TABLE_NAME_SHORT = "a";
+ private static final String APP_TO_FLOW_TABLE_NAME_SHORT = "a2f";
+ private static final String TTL_OPTION_SHORT = "m";
+ private static final String ENTITY_TABLE_NAME_SHORT = "e";
public static void main(String[] args) throws Exception {
@@ -66,22 +71,25 @@ public class TimelineSchemaCreator {
CommandLine commandLine = parseArgs(otherArgs);
// Grab the entityTableName argument
- String entityTableName = commandLine.getOptionValue("e");
+ String entityTableName
+ = commandLine.getOptionValue(ENTITY_TABLE_NAME_SHORT);
if (StringUtils.isNotBlank(entityTableName)) {
hbaseConf.set(EntityTable.TABLE_NAME_CONF_NAME, entityTableName);
}
- String entityTableTTLMetrics = commandLine.getOptionValue("m");
+ String entityTableTTLMetrics = commandLine.getOptionValue(TTL_OPTION_SHORT);
if (StringUtils.isNotBlank(entityTableTTLMetrics)) {
int metricsTTL = Integer.parseInt(entityTableTTLMetrics);
new EntityTable().setMetricsTTL(metricsTTL, hbaseConf);
}
// Grab the appToflowTableName argument
- String appToflowTableName = commandLine.getOptionValue("a2f");
+ String appToflowTableName = commandLine.getOptionValue(
+ APP_TO_FLOW_TABLE_NAME_SHORT);
if (StringUtils.isNotBlank(appToflowTableName)) {
hbaseConf.set(AppToFlowTable.TABLE_NAME_CONF_NAME, appToflowTableName);
}
// Grab the applicationTableName argument
- String applicationTableName = commandLine.getOptionValue("a");
+ String applicationTableName = commandLine.getOptionValue(
+ APP_TABLE_NAME_SHORT);
if (StringUtils.isNotBlank(applicationTableName)) {
hbaseConf.set(ApplicationTable.TABLE_NAME_CONF_NAME,
applicationTableName);
@@ -89,7 +97,13 @@ public class TimelineSchemaCreator {
List<Exception> exceptions = new ArrayList<>();
try {
- createAllTables(hbaseConf);
+ boolean skipExisting
+ = commandLine.hasOption(SKIP_EXISTING_TABLE_OPTION_SHORT);
+ if (skipExisting) {
+ LOG.info("Will skip existing tables and continue on htable creation "
+ + "exceptions!");
+ }
+ createAllTables(hbaseConf, skipExisting);
LOG.info("Successfully created HBase schema. ");
} catch (IOException e) {
LOG.error("Error in creating hbase tables: " + e.getMessage());
@@ -135,26 +149,39 @@ public class TimelineSchemaCreator {
Options options = new Options();
// Input
- Option o = new Option("e", "entityTableName", true, "entity table name");
+ Option o = new Option(ENTITY_TABLE_NAME_SHORT, "entityTableName", true,
+ "entity table name");
o.setArgName("entityTableName");
o.setRequired(false);
options.addOption(o);
- o = new Option("m", "metricsTTL", true, "TTL for metrics column family");
+ o = new Option(TTL_OPTION_SHORT, "metricsTTL", true,
+ "TTL for metrics column family");
o.setArgName("metricsTTL");
o.setRequired(false);
options.addOption(o);
- o = new Option("a2f", "appToflowTableName", true, "app to flow table name");
+ o = new Option(APP_TO_FLOW_TABLE_NAME_SHORT, "appToflowTableName", true,
+ "app to flow table name");
o.setArgName("appToflowTableName");
- o = new Option("a", "applicationTableName", true, "application table name");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option(APP_TABLE_NAME_SHORT, "applicationTableName", true,
+ "application table name");
o.setArgName("applicationTableName");
o.setRequired(false);
options.addOption(o);
+ // Options without an argument
+ // No need to set arg name since we do not need an argument here
o = new Option(PHOENIX_OPTION_SHORT, "usePhoenix", false,
"create Phoenix offline aggregation tables");
- // No need to set arg name since we do not need an argument here
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option(SKIP_EXISTING_TABLE_OPTION_SHORT, "skipExistingTable",
+ false, "skip existing Hbase tables and continue to create new tables");
o.setRequired(false);
options.addOption(o);
@@ -172,8 +199,8 @@ public class TimelineSchemaCreator {
return commandLine;
}
- private static void createAllTables(Configuration hbaseConf)
- throws IOException {
+ private static void createAllTables(Configuration hbaseConf,
+ boolean skipExisting) throws IOException {
Connection conn = null;
try {
@@ -182,9 +209,33 @@ public class TimelineSchemaCreator {
if (admin == null) {
throw new IOException("Cannot create table since admin is null");
}
- new EntityTable().createTable(admin, hbaseConf);
- new AppToFlowTable().createTable(admin, hbaseConf);
- new ApplicationTable().createTable(admin, hbaseConf);
+ try {
+ new EntityTable().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
+ try {
+ new AppToFlowTable().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
+ try {
+ new ApplicationTable().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
} finally {
if (conn != null) {
conn.close();
[14/50] [abbrv] hadoop git commit: YARN-3044. Made RM write app,
attempt and optional container lifecycle events to timeline service
v2. Contributed by Naganarasimha G R.
Posted by sj...@apache.org.
YARN-3044. Made RM write app, attempt and optional container lifecycle events to timeline service v2. Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d35d861c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d35d861c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d35d861c
Branch: refs/heads/YARN-2928-rebase
Commit: d35d861c7990e7680a32f1123c8a97ed932d45a4
Parents: ec0fd03
Author: Zhijie Shen <zj...@apache.org>
Authored: Sat Jun 13 11:32:41 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:29 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/timelineservice/TimelineEntity.java | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +
.../distributedshell/TestDistributedShell.java | 118 +++--
.../metrics/ContainerMetricsConstants.java | 10 +
.../server/resourcemanager/ResourceManager.java | 2 +-
.../AbstractTimelineServicePublisher.java | 184 ++++++++
.../metrics/ApplicationFinishedEvent.java | 13 +-
.../metrics/SystemMetricsPublisher.java | 458 ++++---------------
.../metrics/TimelineServiceV1Publisher.java | 286 ++++++++++++
.../metrics/TimelineServiceV2Publisher.java | 318 +++++++++++++
.../server/resourcemanager/rmapp/RMAppImpl.java | 2 -
.../TestRMAppLogAggregationStatus.java | 2 +-
.../metrics/TestSystemMetricsPublisher.java | 5 +-
.../TestSystemMetricsPublisherForV2.java | 374 +++++++++++++++
.../storage/FileSystemTimelineWriterImpl.java | 4 +-
16 files changed, 1369 insertions(+), 423 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8ff151a..22860ea 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -82,6 +82,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3721. build is broken on YARN-2928 branch due to possible dependency
cycle (Li Lu via sjlee)
+ YARN-3044. Made RM write app, attempt and optional container lifecycle
+ events to timeline service v2. (Naganarasimha G R via zjshen)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index defadec..a641f32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -470,4 +470,7 @@ public class TimelineEntity {
return real == null ? this : real;
}
+ public String toString() {
+ return identifier.toString();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 348eadd..524203b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -392,6 +392,16 @@ public class YarnConfiguration extends Configuration {
+ "system-metrics-publisher.enabled";
public static final boolean DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED = false;
+ /**
+ * The setting that controls whether yarn container metrics is published to
+ * the timeline server or not by RM. This configuration setting is for ATS
+ * V2
+ */
+ public static final String RM_PUBLISH_CONTAINER_METRICS_ENABLED = YARN_PREFIX
+ + "rm.system-metrics-publisher.emit-container-events";
+ public static final boolean DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED =
+ false;
+
public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
RM_PREFIX + "system-metrics-publisher.dispatcher.pool-size";
public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index cc5f5e2..d31f2c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -52,13 +52,14 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -107,6 +108,7 @@ public class TestDistributedShell {
conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
+ ".class", PerNodeTimelineCollectorsAuxService.class.getName());
}
+ conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
// Enable ContainersMonitorImpl
@@ -373,48 +375,96 @@ public class TestDistributedShell {
"/1/1/" : "/test_flow_name/test_flow_version/12345678/") +
appId.toString();
// for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
- String outputDirApp = basePath + "/DS_APP_ATTEMPT/";
-
- File entityFolder = new File(outputDirApp);
- Assert.assertTrue(entityFolder.isDirectory());
+ // Verify DS_APP_ATTEMPT entities posted by the client
// there will be at least one attempt, look for that file
- String appTimestampFileName = "appattempt_" + appId.getClusterTimestamp()
- + "_000" + appId.getId() + "_000001"
- + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
- String appAttemptFileName = outputDirApp + appTimestampFileName;
- File appAttemptFile = new File(appAttemptFileName);
- Assert.assertTrue(appAttemptFile.exists());
-
- String outputDirContainer = basePath + "/DS_CONTAINER/";
- File containerFolder = new File(outputDirContainer);
- Assert.assertTrue(containerFolder.isDirectory());
-
- String containerTimestampFileName = "container_"
- + appId.getClusterTimestamp() + "_000" + appId.getId()
- + "_01_000002.thist";
- String containerFileName = outputDirContainer + containerTimestampFileName;
- File containerFile = new File(containerFileName);
- Assert.assertTrue(containerFile.exists());
+ String appTimestampFileName =
+ "appattempt_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + "_000001"
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ verifyEntityTypeFileExists(basePath, "DS_APP_ATTEMPT",
+ appTimestampFileName);
+
+ // Verify DS_CONTAINER entities posted by the client
+ String containerTimestampFileName =
+ "container_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + "_01_000002.thist";
+ verifyEntityTypeFileExists(basePath, "DS_CONTAINER",
+ containerTimestampFileName);
// Verify NM posting container metrics info.
- String outputDirContainerMetrics = basePath + "/" +
- TimelineEntityType.YARN_CONTAINER + "/";
- File containerMetricsFolder = new File(outputDirContainerMetrics);
- Assert.assertTrue(containerMetricsFolder.isDirectory());
+ String containerMetricsTimestampFileName =
+ "container_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + "_01_000001"
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ verifyEntityTypeFileExists(basePath,
+ TimelineEntityType.YARN_CONTAINER.toString(),
+ containerMetricsTimestampFileName);
+
+ // Verify RM posting Application life cycle Events are getting published
+ String appMetricsTimestampFileName =
+ "application_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ File appEntityFile =
+ verifyEntityTypeFileExists(basePath,
+ TimelineEntityType.YARN_APPLICATION.toString(),
+ appMetricsTimestampFileName);
+ verifyStringExistsSpecifiedTimes(appEntityFile,
+ ApplicationMetricsConstants.CREATED_EVENT_TYPE, 1,
+ "Application created event should be published atleast once");
+ verifyStringExistsSpecifiedTimes(appEntityFile,
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE, 1,
+ "Application finished event should be published atleast once");
+
+ // Verify RM posting AppAttempt life cycle Events are getting published
+ String appAttemptMetricsTimestampFileName =
+ "appattempt_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + "_000001"
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ File appAttemptEntityFile =
+ verifyEntityTypeFileExists(basePath,
+ TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
+ appAttemptMetricsTimestampFileName);
+ verifyStringExistsSpecifiedTimes(appAttemptEntityFile,
+ AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE, 1,
+ "AppAttempt register event should be published atleast once");
+ verifyStringExistsSpecifiedTimes(appAttemptEntityFile,
+ AppAttemptMetricsConstants.FINISHED_EVENT_TYPE, 1,
+ "AppAttempt finished event should be published atleast once");
+ } finally {
+ FileUtils.deleteDirectory(tmpRootFolder.getParentFile());
+ }
+ }
- String containerMetricsTimestampFileName = "container_"
- + appId.getClusterTimestamp() + "_000" + appId.getId()
- + "_01_000001.thist";
- String containerMetricsFileName = outputDirContainerMetrics +
- containerMetricsTimestampFileName;
+ private File verifyEntityTypeFileExists(String basePath, String entityType,
+ String entityfileName) {
+ String outputDirPathForEntity = basePath + "/" + entityType + "/";
+ File outputDirForEntity = new File(outputDirPathForEntity);
+ Assert.assertTrue(outputDirForEntity.isDirectory());
- File containerMetricsFile = new File(containerMetricsFileName);
- Assert.assertTrue(containerMetricsFile.exists());
+ String entityFilePath = outputDirPathForEntity + entityfileName;
+ File entityFile = new File(entityFilePath);
+ Assert.assertTrue(entityFile.exists());
+ return entityFile;
+ }
+
+ private void verifyStringExistsSpecifiedTimes(File entityFile,
+ String searchString, long expectedNumOfTimes, String errorMsg)
+ throws IOException {
+ BufferedReader reader = null;
+ String strLine;
+ long actualCount = 0;
+ try {
+ reader = new BufferedReader(new FileReader(entityFile));
+ while ((strLine = reader.readLine()) != null) {
+ if (strLine.trim().contains(searchString))
+ actualCount++;
+ }
} finally {
- FileUtils.deleteDirectory(tmpRootFolder.getParentFile());
+ reader.close();
}
+ Assert.assertEquals(errorMsg, expectedNumOfTimes, actualCount);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
index 0d5540d..7b42994 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
@@ -27,10 +27,20 @@ public class ContainerMetricsConstants {
public static final String ENTITY_TYPE = "YARN_CONTAINER";
+ // Event of this type will be emitted by NM.
public static final String CREATED_EVENT_TYPE = "YARN_CONTAINER_CREATED";
+ // Event of this type will be emitted by RM.
+ public static final String CREATED_IN_RM_EVENT_TYPE =
+ "YARN_RM_CONTAINER_CREATED";
+
+ // Event of this type will be emitted by NM.
public static final String FINISHED_EVENT_TYPE = "YARN_CONTAINER_FINISHED";
+ // Event of this type will be emitted by RM.
+ public static final String FINISHED_IN_RM_EVENT_TYPE =
+ "YARN_RM_CONTAINER_FINISHED";
+
public static final String PARENT_PRIMARIY_FILTER = "YARN_CONTAINER_PARENT";
public static final String ALLOCATED_MEMORY_ENTITY_INFO =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 29262ea..93049f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -373,7 +373,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
protected SystemMetricsPublisher createSystemMetricsPublisher() {
- return new SystemMetricsPublisher();
+ return new SystemMetricsPublisher(rmContext);
}
// sanity check for configurations
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractTimelineServicePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractTimelineServicePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractTimelineServicePublisher.java
new file mode 100644
index 0000000..12145267
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractTimelineServicePublisher.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher.TimelineServicePublisher;
+
+public abstract class AbstractTimelineServicePublisher extends CompositeService
+ implements TimelineServicePublisher, EventHandler<SystemMetricsEvent> {
+
+ private static final Log LOG = LogFactory
+ .getLog(TimelineServiceV2Publisher.class);
+
+ private Configuration conf;
+
+ public AbstractTimelineServicePublisher(String name) {
+ super(name);
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ this.conf = conf;
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ }
+
+ @Override
+ public void handle(SystemMetricsEvent event) {
+ switch (event.getType()) {
+ case APP_CREATED:
+ publishApplicationCreatedEvent((ApplicationCreatedEvent) event);
+ break;
+ case APP_FINISHED:
+ publishApplicationFinishedEvent((ApplicationFinishedEvent) event);
+ break;
+ case APP_UPDATED:
+ publishApplicationUpdatedEvent((ApplicationUpdatedEvent) event);
+ break;
+ case APP_ACLS_UPDATED:
+ publishApplicationACLsUpdatedEvent((ApplicationACLsUpdatedEvent) event);
+ break;
+ case APP_ATTEMPT_REGISTERED:
+ publishAppAttemptRegisteredEvent((AppAttemptRegisteredEvent) event);
+ break;
+ case APP_ATTEMPT_FINISHED:
+ publishAppAttemptFinishedEvent((AppAttemptFinishedEvent) event);
+ break;
+ case CONTAINER_CREATED:
+ publishContainerCreatedEvent((ContainerCreatedEvent) event);
+ break;
+ case CONTAINER_FINISHED:
+ publishContainerFinishedEvent((ContainerFinishedEvent) event);
+ break;
+ default:
+ LOG.error("Unknown SystemMetricsEvent type: " + event.getType());
+ }
+ }
+
+ abstract void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event);
+
+ abstract void publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event);
+
+ abstract void publishApplicationUpdatedEvent(ApplicationUpdatedEvent event);
+
+ abstract void publishApplicationACLsUpdatedEvent(
+ ApplicationACLsUpdatedEvent event);
+
+ abstract void publishApplicationFinishedEvent(ApplicationFinishedEvent event);
+
+ abstract void publishApplicationCreatedEvent(ApplicationCreatedEvent event);
+
+ abstract void publishContainerCreatedEvent(ContainerCreatedEvent event);
+
+ abstract void publishContainerFinishedEvent(ContainerFinishedEvent event);
+
+ @Override
+ public Dispatcher getDispatcher() {
+ MultiThreadedDispatcher dispatcher =
+ new MultiThreadedDispatcher(
+ conf.getInt(
+ YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
+ YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE));
+ dispatcher.setDrainEventsOnStop();
+ return dispatcher;
+ }
+
+ @Override
+ public boolean publishRMContainerMetrics() {
+ return true;
+ }
+
+ @Override
+ public EventHandler<SystemMetricsEvent> getEventHandler() {
+ return this;
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public static class MultiThreadedDispatcher extends CompositeService
+ implements Dispatcher {
+
+ private List<AsyncDispatcher> dispatchers =
+ new ArrayList<AsyncDispatcher>();
+
+ public MultiThreadedDispatcher(int num) {
+ super(MultiThreadedDispatcher.class.getName());
+ for (int i = 0; i < num; ++i) {
+ AsyncDispatcher dispatcher = createDispatcher();
+ dispatchers.add(dispatcher);
+ addIfService(dispatcher);
+ }
+ }
+
+ @Override
+ public EventHandler getEventHandler() {
+ return new CompositEventHandler();
+ }
+
+ @Override
+ public void register(Class<? extends Enum> eventType, EventHandler handler) {
+ for (AsyncDispatcher dispatcher : dispatchers) {
+ dispatcher.register(eventType, handler);
+ }
+ }
+
+ public void setDrainEventsOnStop() {
+ for (AsyncDispatcher dispatcher : dispatchers) {
+ dispatcher.setDrainEventsOnStop();
+ }
+ }
+
+ private class CompositEventHandler implements EventHandler<Event> {
+
+ @Override
+ public void handle(Event event) {
+ // Use hashCode (of ApplicationId) to dispatch the event to the child
+ // dispatcher, such that all the writing events of one application will
+ // be handled by one thread, the scheduled order of the these events
+ // will be preserved
+ int index = (event.hashCode() & Integer.MAX_VALUE) % dispatchers.size();
+ dispatchers.get(index).getEventHandler().handle(event);
+ }
+ }
+
+ protected AsyncDispatcher createDispatcher() {
+ return new AsyncDispatcher();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
index 8d75f92..d9241b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
public class ApplicationFinishedEvent extends
@@ -33,6 +35,7 @@ public class ApplicationFinishedEvent extends
private YarnApplicationState state;
private ApplicationAttemptId latestAppAttemptId;
private RMAppMetrics appMetrics;
+ private RMAppImpl app;
public ApplicationFinishedEvent(
ApplicationId appId,
@@ -41,14 +44,16 @@ public class ApplicationFinishedEvent extends
YarnApplicationState state,
ApplicationAttemptId latestAppAttemptId,
long finishedTime,
- RMAppMetrics appMetrics) {
+ RMAppMetrics appMetrics,
+ RMAppImpl app) {
super(SystemMetricsEventType.APP_FINISHED, finishedTime);
this.appId = appId;
this.diagnosticsInfo = diagnosticsInfo;
this.appStatus = appStatus;
this.latestAppAttemptId = latestAppAttemptId;
this.state = state;
- this.appMetrics=appMetrics;
+ this.appMetrics = appMetrics;
+ this.app = app;
}
@Override
@@ -56,6 +61,10 @@ public class ApplicationFinishedEvent extends
return appId.hashCode();
}
+ public RMAppImpl getApp() {
+ return app;
+ }
+
public ApplicationId getApplicationId() {
return appId;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 2f56bd7..6ab7838 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -18,10 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.metrics;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -29,33 +25,24 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
-import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
-import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+import com.google.common.annotations.VisibleForTesting;
/**
- * The class that helps RM publish metrics to the timeline server V1. RM will
+ * The class that helps RM publish metrics to the timeline server. RM will
* always invoke the methods of this class regardless the service is enabled or
* not. If it is disabled, publishing requests will be ignored silently.
*/
@@ -67,30 +54,38 @@ public class SystemMetricsPublisher extends CompositeService {
.getLog(SystemMetricsPublisher.class);
private Dispatcher dispatcher;
- private TimelineClient client;
- private boolean publishSystemMetricsToATSv1;
+ private boolean publishSystemMetrics;
+ private boolean publishContainerMetrics;
+ protected RMContext rmContext;
- public SystemMetricsPublisher() {
+ public SystemMetricsPublisher(RMContext rmContext) {
super(SystemMetricsPublisher.class.getName());
+ this.rmContext = rmContext;
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
- publishSystemMetricsToATSv1 =
+ publishSystemMetrics =
conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)
- && conf.getBoolean(
- YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED,
- YarnConfiguration.DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED);
-
- if (publishSystemMetricsToATSv1) {
- client = TimelineClient.createTimelineClient();
- addIfService(client);
-
- dispatcher = createDispatcher(conf);
- dispatcher.register(SystemMetricsEventType.class,
- new ForwardingEventHandler());
- addIfService(dispatcher);
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED);
+ if (publishSystemMetrics) {
+ TimelineServicePublisher timelineServicePublisher =
+ getTimelineServicePublisher(conf);
+ if (timelineServicePublisher != null) {
+ addService(timelineServicePublisher);
+ // init required to be called so that other methods of
+ // TimelineServicePublisher can be utilized
+ timelineServicePublisher.init(conf);
+ dispatcher = createDispatcher(timelineServicePublisher);
+ publishContainerMetrics =
+ timelineServicePublisher.publishRMContainerMetrics();
+ dispatcher.register(SystemMetricsEventType.class,
+ timelineServicePublisher.getEventHandler());
+ addIfService(dispatcher);
+ } else {
+ LOG.info("TimelineServicePublisher is not configured");
+ publishSystemMetrics = false;
+ }
LOG.info("YARN system metrics publishing service is enabled");
} else {
LOG.info("YARN system metrics publishing service is not enabled");
@@ -98,9 +93,26 @@ public class SystemMetricsPublisher extends CompositeService {
super.serviceInit(conf);
}
+ @VisibleForTesting
+ Dispatcher createDispatcher(TimelineServicePublisher timelineServicePublisher) {
+ return timelineServicePublisher.getDispatcher();
+ }
+
+ TimelineServicePublisher getTimelineServicePublisher(Configuration conf) {
+ if (conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
+ YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED)) {
+ return new TimelineServiceV1Publisher();
+ } else if (conf.getBoolean(
+ YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED,
+ YarnConfiguration.DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED)) {
+ return new TimelineServiceV2Publisher(rmContext);
+ }
+ return null;
+ }
+
@SuppressWarnings("unchecked")
public void appCreated(RMApp app, long createdTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
ApplicationSubmissionContext appSubmissionContext =
app.getApplicationSubmissionContext();
dispatcher.getEventHandler().handle(
@@ -121,7 +133,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appUpdated(RMApp app, long updatedTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
dispatcher.getEventHandler()
.handle(
new ApplicationUpdatedEvent(app.getApplicationId(), app
@@ -132,7 +144,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appFinished(RMApp app, RMAppState state, long finishedTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new ApplicationFinishedEvent(
app.getApplicationId(),
@@ -142,14 +154,15 @@ public class SystemMetricsPublisher extends CompositeService {
app.getCurrentAppAttempt() == null ?
null : app.getCurrentAppAttempt().getAppAttemptId(),
finishedTime,
- app.getRMAppMetrics()));
+ app.getRMAppMetrics(),
+ (RMAppImpl)app));
}
}
@SuppressWarnings("unchecked")
public void appACLsUpdated(RMApp app, String appViewACLs,
long updatedTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new ApplicationACLsUpdatedEvent(
app.getApplicationId(),
@@ -161,7 +174,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appAttemptRegistered(RMAppAttempt appAttempt,
long registeredTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new AppAttemptRegisteredEvent(
appAttempt.getAppAttemptId(),
@@ -177,7 +190,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appAttemptFinished(RMAppAttempt appAttempt,
RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new AppAttemptFinishedEvent(
appAttempt.getAppAttemptId(),
@@ -194,7 +207,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void containerCreated(RMContainer container, long createdTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishContainerMetrics) {
dispatcher.getEventHandler().handle(
new ContainerCreatedEvent(
container.getContainerId(),
@@ -207,7 +220,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void containerFinished(RMContainer container, long finishedTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishContainerMetrics) {
dispatcher.getEventHandler().handle(
new ContainerFinishedEvent(
container.getContainerId(),
@@ -218,344 +231,31 @@ public class SystemMetricsPublisher extends CompositeService {
}
}
- protected Dispatcher createDispatcher(Configuration conf) {
- MultiThreadedDispatcher dispatcher =
- new MultiThreadedDispatcher(
- conf.getInt(
- YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
- YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE));
- dispatcher.setDrainEventsOnStop();
- return dispatcher;
- }
-
- protected void handleSystemMetricsEvent(
- SystemMetricsEvent event) {
- switch (event.getType()) {
- case APP_CREATED:
- publishApplicationCreatedEvent((ApplicationCreatedEvent) event);
- break;
- case APP_FINISHED:
- publishApplicationFinishedEvent((ApplicationFinishedEvent) event);
- break;
- case APP_ACLS_UPDATED:
- publishApplicationACLsUpdatedEvent((ApplicationACLsUpdatedEvent) event);
- break;
- case APP_UPDATED:
- publishApplicationUpdatedEvent((ApplicationUpdatedEvent) event);
- break;
- case APP_ATTEMPT_REGISTERED:
- publishAppAttemptRegisteredEvent((AppAttemptRegisteredEvent) event);
- break;
- case APP_ATTEMPT_FINISHED:
- publishAppAttemptFinishedEvent((AppAttemptFinishedEvent) event);
- break;
- case CONTAINER_CREATED:
- publishContainerCreatedEvent((ContainerCreatedEvent) event);
- break;
- case CONTAINER_FINISHED:
- publishContainerFinishedEvent((ContainerFinishedEvent) event);
- break;
- default:
- LOG.error("Unknown SystemMetricsEvent type: " + event.getType());
- }
- }
-
- private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
- TimelineEntity entity =
- createApplicationEntity(event.getApplicationId());
- Map<String, Object> entityInfo = new HashMap<String, Object>();
- entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO,
- event.getApplicationName());
- entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
- event.getApplicationType());
- entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO,
- event.getUser());
- entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
- event.getQueue());
- entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
- event.getSubmittedTime());
- entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
- event.getAppTags());
- entityInfo.put(
- ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO,
- event.isUnmanagedApp());
- entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
- event.getApplicationPriority().getPriority());
- entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
- event.getAppNodeLabelsExpression());
- entityInfo.put(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION,
- event.getAmNodeLabelsExpression());
- entity.setOtherInfo(entityInfo);
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(
- ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
- TimelineEntity entity =
- createApplicationEntity(event.getApplicationId());
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(
- ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- Map<String, Object> eventInfo = new HashMap<String, Object>();
- eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
- event.getDiagnosticsInfo());
- eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
- event.getFinalApplicationStatus().toString());
- eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
- event.getYarnApplicationState().toString());
- if (event.getLatestApplicationAttemptId() != null) {
- eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
- event.getLatestApplicationAttemptId().toString());
- }
- RMAppMetrics appMetrics = event.getAppMetrics();
- entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
- appMetrics.getVcoreSeconds());
- entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
- appMetrics.getMemorySeconds());
-
- tEvent.setEventInfo(eventInfo);
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private void publishApplicationUpdatedEvent(ApplicationUpdatedEvent event) {
- TimelineEntity entity = createApplicationEntity(event.getApplicationId());
- Map<String, Object> eventInfo = new HashMap<String, Object>();
- eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
- event.getQueue());
- eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, event
- .getApplicationPriority().getPriority());
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(ApplicationMetricsConstants.UPDATED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- tEvent.setEventInfo(eventInfo);
- entity.addEvent(tEvent);
- putEntity(entity);
+ @VisibleForTesting
+ boolean isPublishContainerMetrics() {
+ return publishContainerMetrics;
}
- private void publishApplicationACLsUpdatedEvent(
- ApplicationACLsUpdatedEvent event) {
- TimelineEntity entity =
- createApplicationEntity(event.getApplicationId());
- TimelineEvent tEvent = new TimelineEvent();
- Map<String, Object> entityInfo = new HashMap<String, Object>();
- entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
- event.getViewAppACLs());
- entity.setOtherInfo(entityInfo);
- tEvent.setEventType(
- ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private static TimelineEntity createApplicationEntity(
- ApplicationId applicationId) {
- TimelineEntity entity = new TimelineEntity();
- entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
- entity.setEntityId(applicationId.toString());
- return entity;
- }
-
- private void
- publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event) {
- TimelineEntity entity =
- createAppAttemptEntity(event.getApplicationAttemptId());
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(
- AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- Map<String, Object> eventInfo = new HashMap<String, Object>();
- eventInfo.put(
- AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
- event.getTrackingUrl());
- eventInfo.put(
- AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
- event.getOriginalTrackingURL());
- eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO,
- event.getHost());
- eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
- event.getRpcPort());
- eventInfo.put(
- AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
- event.getMasterContainerId().toString());
- tEvent.setEventInfo(eventInfo);
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) {
- TimelineEntity entity =
- createAppAttemptEntity(event.getApplicationAttemptId());
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- Map<String, Object> eventInfo = new HashMap<String, Object>();
- eventInfo.put(
- AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
- event.getTrackingUrl());
- eventInfo.put(
- AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
- event.getOriginalTrackingURL());
- eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
- event.getDiagnosticsInfo());
- eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO,
- event.getFinalApplicationStatus().toString());
- eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO,
- event.getYarnApplicationAttemptState().toString());
- tEvent.setEventInfo(eventInfo);
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private static TimelineEntity createAppAttemptEntity(
- ApplicationAttemptId appAttemptId) {
- TimelineEntity entity = new TimelineEntity();
- entity.setEntityType(
- AppAttemptMetricsConstants.ENTITY_TYPE);
- entity.setEntityId(appAttemptId.toString());
- entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER,
- appAttemptId.getApplicationId().toString());
- return entity;
- }
-
- private void publishContainerCreatedEvent(ContainerCreatedEvent event) {
- TimelineEntity entity = createContainerEntity(event.getContainerId());
- Map<String, Object> entityInfo = new HashMap<String, Object>();
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO,
- event.getAllocatedResource().getMemory());
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO,
- event.getAllocatedResource().getVirtualCores());
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
- event.getAllocatedNode().getHost());
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO,
- event.getAllocatedNode().getPort());
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
- event.getAllocatedPriority().getPriority());
- entityInfo.put(
- ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
- event.getNodeHttpAddress());
- entity.setOtherInfo(entityInfo);
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private void publishContainerFinishedEvent(ContainerFinishedEvent event) {
- TimelineEntity entity = createContainerEntity(event.getContainerId());
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- Map<String, Object> eventInfo = new HashMap<String, Object>();
- eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
- event.getDiagnosticsInfo());
- eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO,
- event.getContainerExitStatus());
- eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO,
- event.getContainerState().toString());
- tEvent.setEventInfo(eventInfo);
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private static TimelineEntity createContainerEntity(
- ContainerId containerId) {
- TimelineEntity entity = new TimelineEntity();
- entity.setEntityType(
- ContainerMetricsConstants.ENTITY_TYPE);
- entity.setEntityId(containerId.toString());
- entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
- containerId.getApplicationAttemptId().toString());
- return entity;
- }
-
- private void putEntity(TimelineEntity entity) {
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Publishing the entity " + entity.getEntityId() +
- ", JSON-style content: " + TimelineUtils.dumpTimelineRecordtoJSON(entity));
- }
- client.putEntities(entity);
- } catch (Exception e) {
- LOG.error("Error when publishing entity [" + entity.getEntityType() + ","
- + entity.getEntityId() + "]", e);
- }
+ @VisibleForTesting
+ Dispatcher getDispatcher() {
+ return dispatcher;
}
- /**
- * EventHandler implementation which forward events to SystemMetricsPublisher.
- * Making use of it, SystemMetricsPublisher can avoid to have a public handle
- * method.
- */
- private final class ForwardingEventHandler implements
- EventHandler<SystemMetricsEvent> {
-
- @Override
- public void handle(SystemMetricsEvent event) {
- handleSystemMetricsEvent(event);
- }
-
+ interface TimelineServicePublisher extends Service {
+ /**
+ * @return the Dispatcher which needs to be used to dispatch events
+ */
+ Dispatcher getDispatcher();
+
+ /**
+ * @return true if RMContainerMetricsNeeds to be sent
+ */
+ boolean publishRMContainerMetrics();
+
+ /**
+ * @return EventHandler which needs to be registered to the dispatcher to
+ * handle the SystemMetricsEvent
+ */
+ EventHandler<SystemMetricsEvent> getEventHandler();
}
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- protected static class MultiThreadedDispatcher extends CompositeService
- implements Dispatcher {
-
- private List<AsyncDispatcher> dispatchers =
- new ArrayList<AsyncDispatcher>();
-
- public MultiThreadedDispatcher(int num) {
- super(MultiThreadedDispatcher.class.getName());
- for (int i = 0; i < num; ++i) {
- AsyncDispatcher dispatcher = createDispatcher();
- dispatchers.add(dispatcher);
- addIfService(dispatcher);
- }
- }
-
- @Override
- public EventHandler getEventHandler() {
- return new CompositEventHandler();
- }
-
- @Override
- public void register(Class<? extends Enum> eventType, EventHandler handler) {
- for (AsyncDispatcher dispatcher : dispatchers) {
- dispatcher.register(eventType, handler);
- }
- }
-
- public void setDrainEventsOnStop() {
- for (AsyncDispatcher dispatcher : dispatchers) {
- dispatcher.setDrainEventsOnStop();
- }
- }
-
- private class CompositEventHandler implements EventHandler<Event> {
-
- @Override
- public void handle(Event event) {
- // Use hashCode (of ApplicationId) to dispatch the event to the child
- // dispatcher, such that all the writing events of one application will
- // be handled by one thread, the scheduled order of the these events
- // will be preserved
- int index = (event.hashCode() & Integer.MAX_VALUE) % dispatchers.size();
- dispatchers.get(index).getEventHandler().handle(event);
- }
-
- }
-
- protected AsyncDispatcher createDispatcher() {
- return new AsyncDispatcher();
- }
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
new file mode 100644
index 0000000..0dd1bca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -0,0 +1,286 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+public class TimelineServiceV1Publisher extends
+ AbstractTimelineServicePublisher {
+
+ private static final Log LOG = LogFactory
+ .getLog(TimelineServiceV1Publisher.class);
+
+ public TimelineServiceV1Publisher() {
+ super("TimelineserviceV1Publisher");
+ }
+
+ private TimelineClient client;
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ client = TimelineClient.createTimelineClient();
+ addIfService(client);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
+ TimelineEntity entity = createApplicationEntity(event.getApplicationId());
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO,
+ event.getApplicationName());
+ entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
+ event.getApplicationType());
+ entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO,
+ event.getUser());
+ entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
+ event.getQueue());
+ entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
+ event.getSubmittedTime());
+ entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
+ event.getAppTags());
+ entityInfo.put(
+ ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO,
+ event.isUnmanagedApp());
+ entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
+ event.getApplicationPriority().getPriority());
+ entity.setOtherInfo(entityInfo);
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ @Override
+ void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
+ TimelineEntity entity = createApplicationEntity(event.getApplicationId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, event
+ .getFinalApplicationStatus().toString());
+ eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, event
+ .getYarnApplicationState().toString());
+ if (event.getLatestApplicationAttemptId() != null) {
+ eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
+ event.getLatestApplicationAttemptId().toString());
+ }
+ RMAppMetrics appMetrics = event.getAppMetrics();
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
+ appMetrics.getVcoreSeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
+ appMetrics.getMemorySeconds());
+ tEvent.setEventInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ @Override
+ void publishApplicationUpdatedEvent(ApplicationUpdatedEvent event) {
+ TimelineEntity entity = createApplicationEntity(event.getApplicationId());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
+ event.getQueue());
+ eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, event
+ .getApplicationPriority().getPriority());
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ApplicationMetricsConstants.UPDATED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ tEvent.setEventInfo(eventInfo);
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ @Override
+ void publishApplicationACLsUpdatedEvent(ApplicationACLsUpdatedEvent event) {
+ TimelineEntity entity = createApplicationEntity(event.getApplicationId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
+ event.getViewAppACLs());
+ entity.setOtherInfo(entityInfo);
+ tEvent.setEventType(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ private static TimelineEntity createApplicationEntity(
+ ApplicationId applicationId) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
+ entity.setEntityId(applicationId.toString());
+ return entity;
+ }
+
+ @Override
+ void publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event) {
+ TimelineEntity entity =
+ createAppAttemptEntity(event.getApplicationAttemptId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
+ event.getTrackingUrl());
+ eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
+ event.getOriginalTrackingURL());
+ eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, event.getHost());
+ eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
+ event.getRpcPort());
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, event
+ .getMasterContainerId().toString());
+ tEvent.setEventInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ @Override
+ void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) {
+ TimelineEntity entity =
+ createAppAttemptEntity(event.getApplicationAttemptId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
+ event.getTrackingUrl());
+ eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
+ event.getOriginalTrackingURL());
+ eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, event
+ .getFinalApplicationStatus().toString());
+ eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, event
+ .getYarnApplicationAttemptState().toString());
+ tEvent.setEventInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ private static TimelineEntity createAppAttemptEntity(
+ ApplicationAttemptId appAttemptId) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setEntityType(AppAttemptMetricsConstants.ENTITY_TYPE);
+ entity.setEntityId(appAttemptId.toString());
+ entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER,
+ appAttemptId.getApplicationId().toString());
+ return entity;
+ }
+
+ @Override
+ void publishContainerCreatedEvent(ContainerCreatedEvent event) {
+ TimelineEntity entity = createContainerEntity(event.getContainerId());
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO,
+ event.getAllocatedResource().getMemory());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, event
+ .getAllocatedResource().getVirtualCores());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, event
+ .getAllocatedNode().getHost());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, event
+ .getAllocatedNode().getPort());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
+ event.getAllocatedPriority().getPriority());
+ entityInfo.put(
+ ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
+ event.getNodeHttpAddress());
+ entity.setOtherInfo(entityInfo);
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ @Override
+ void publishContainerFinishedEvent(ContainerFinishedEvent event) {
+ TimelineEntity entity = createContainerEntity(event.getContainerId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO,
+ event.getContainerExitStatus());
+ eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, event
+ .getContainerState().toString());
+ tEvent.setEventInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ private static TimelineEntity createContainerEntity(ContainerId containerId) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setEntityType(ContainerMetricsConstants.ENTITY_TYPE);
+ entity.setEntityId(containerId.toString());
+ entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
+ containerId.getApplicationAttemptId().toString());
+ return entity;
+ }
+
+ private void putEntity(TimelineEntity entity) {
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Publishing the entity " + entity.getEntityId()
+ + ", JSON-style content: "
+ + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ }
+ client.putEntities(entity);
+ } catch (Exception e) {
+ LOG.error("Error when publishing entity [" + entity.getEntityType() + ","
+ + entity.getEntityId() + "]", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
new file mode 100644
index 0000000..033e0c0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -0,0 +1,318 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+/**
+ * This class is responsible for posting application, appattempt & Container
+ * lifecycle related events to timeline service V2
+ */
+@Private
+@Unstable
+public class TimelineServiceV2Publisher extends
+ AbstractTimelineServicePublisher {
+ private static final Log LOG = LogFactory
+ .getLog(TimelineServiceV2Publisher.class);
+ protected RMTimelineCollectorManager rmTimelineCollectorManager;
+
+ public TimelineServiceV2Publisher(RMContext rmContext) {
+ super("TimelineserviceV2Publisher");
+ rmTimelineCollectorManager = rmContext.getRMTimelineCollectorManager();
+ }
+
+ private boolean publishContainerMetrics;
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ publishContainerMetrics =
+ conf.getBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
+ YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
+ ApplicationEntity entity =
+ createApplicationEntity(event.getApplicationId());
+ entity.setQueue(event.getQueue());
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO,
+ event.getApplicationName());
+ entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
+ event.getApplicationType());
+ entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO,
+ event.getUser());
+ entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
+ event.getSubmittedTime());
+ entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
+ event.getAppTags());
+ entityInfo.put(
+ ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO,
+ event.isUnmanagedApp());
+ entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
+ event.getApplicationPriority().getPriority());
+ entity.setInfo(entityInfo);
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ entity.addEvent(tEvent);
+
+ putEntity(entity, event.getApplicationId());
+ }
+
+ @Override
+ void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
+ ApplicationEntity entity =
+ createApplicationEntity(event.getApplicationId());
+ RMAppMetrics appMetrics = event.getAppMetrics();
+ entity.addInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
+ appMetrics.getVcoreSeconds());
+ entity.addInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
+ appMetrics.getMemorySeconds());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, event
+ .getFinalApplicationStatus().toString());
+ eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, event
+ .getYarnApplicationState().toString());
+ if (event.getLatestApplicationAttemptId() != null) {
+ eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
+ event.getLatestApplicationAttemptId().toString());
+ }
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getApplicationId());
+
+ //cleaning up the collector cached
+ event.getApp().stopTimelineCollector();
+ }
+
+ @Override
+ void publishApplicationUpdatedEvent(ApplicationUpdatedEvent event) {
+ ApplicationEntity entity =
+ createApplicationEntity(event.getApplicationId());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
+ event.getQueue());
+ eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, event
+ .getApplicationPriority().getPriority());
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ApplicationMetricsConstants.UPDATED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ tEvent.setInfo(eventInfo);
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getApplicationId());
+ }
+
+ @Override
+ void publishApplicationACLsUpdatedEvent(ApplicationACLsUpdatedEvent event) {
+ ApplicationEntity entity =
+ createApplicationEntity(event.getApplicationId());
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
+ event.getViewAppACLs());
+ entity.setInfo(entityInfo);
+
+ putEntity(entity, event.getApplicationId());
+ }
+
+ private static ApplicationEntity createApplicationEntity(
+ ApplicationId applicationId) {
+ ApplicationEntity entity = new ApplicationEntity();
+ entity.setId(applicationId.toString());
+ return entity;
+ }
+
+ @Override
+ void publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event) {
+ TimelineEntity entity =
+ createAppAttemptEntity(event.getApplicationAttemptId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
+ event.getTrackingUrl());
+ eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
+ event.getOriginalTrackingURL());
+ eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, event.getHost());
+ eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
+ event.getRpcPort());
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, event
+ .getMasterContainerId().toString());
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getApplicationAttemptId().getApplicationId());
+ }
+
+ @Override
+ void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) {
+ ApplicationAttemptEntity entity =
+ createAppAttemptEntity(event.getApplicationAttemptId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
+ event.getTrackingUrl());
+ eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
+ event.getOriginalTrackingURL());
+ eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, event
+ .getFinalApplicationStatus().toString());
+ eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, event
+ .getYarnApplicationAttemptState().toString());
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getApplicationAttemptId().getApplicationId());
+ }
+
+ @Override
+ void publishContainerCreatedEvent(ContainerCreatedEvent event) {
+ TimelineEntity entity = createContainerEntity(event.getContainerId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ // updated as event info instead of entity info, as entity info is updated
+ // by NM
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO, event
+ .getAllocatedResource().getMemory());
+ eventInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, event
+ .getAllocatedResource().getVirtualCores());
+ eventInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, event
+ .getAllocatedNode().getHost());
+ eventInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, event
+ .getAllocatedNode().getPort());
+ eventInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
+ event.getAllocatedPriority().getPriority());
+ eventInfo.put(
+ ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
+ event.getNodeHttpAddress());
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getContainerId().getApplicationAttemptId()
+ .getApplicationId());
+ }
+
+ @Override
+ void publishContainerFinishedEvent(ContainerFinishedEvent event) {
+ TimelineEntity entity = createContainerEntity(event.getContainerId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ContainerMetricsConstants.FINISHED_IN_RM_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO,
+ event.getContainerExitStatus());
+ eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, event
+ .getContainerState().toString());
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getContainerId().getApplicationAttemptId()
+ .getApplicationId());
+ }
+
+ private static ContainerEntity createContainerEntity(ContainerId containerId) {
+ ContainerEntity entity = new ContainerEntity();
+ entity.setId(containerId.toString());
+ entity.setParent(new Identifier(TimelineEntityType.YARN_APPLICATION_ATTEMPT
+ .name(), containerId.getApplicationAttemptId().toString()));
+ return entity;
+ }
+
+ private void putEntity(TimelineEntity entity, ApplicationId appId) {
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Publishing the entity " + entity + ", JSON-style content: "
+ + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ }
+ TimelineCollector timelineCollector =
+ rmTimelineCollectorManager.get(appId);
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ timelineCollector.putEntities(entities,
+ UserGroupInformation.getCurrentUser());
+ } catch (Exception e) {
+ LOG.error("Error when publishing entity " + entity, e);
+ }
+ }
+
+ private static ApplicationAttemptEntity createAppAttemptEntity(
+ ApplicationAttemptId appAttemptId) {
+ ApplicationAttemptEntity entity = new ApplicationAttemptEntity();
+ entity.setId(appAttemptId.toString());
+ entity.setParent(new Identifier(TimelineEntityType.YARN_APPLICATION.name(),
+ appAttemptId.getApplicationId().toString()));
+ return entity;
+ }
+
+ @Override
+ public boolean publishRMContainerMetrics() {
+ return publishContainerMetrics;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 6bd41e2..7c1d375 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1337,8 +1337,6 @@ public class RMAppImpl implements RMApp, Recoverable {
.applicationFinished(app, finalState);
app.rmContext.getSystemMetricsPublisher()
.appFinished(app, finalState, app.finishTime);
-
- app.stopTimelineCollector();
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
index ec71846..d0bd68f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
@@ -89,7 +89,7 @@ public class TestRMAppLogAggregationStatus {
rmContext =
new RMContextImpl(rmDispatcher, null, null, null,
null, null, null, null, null);
- rmContext.setSystemMetricsPublisher(new SystemMetricsPublisher());
+ rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
rmContext
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d35d861c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 5965ecb..0418f37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistor
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -74,7 +75,7 @@ public class TestSystemMetricsPublisher {
public static void setup() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
MemoryTimelineStore.class, TimelineStore.class);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
@@ -88,7 +89,7 @@ public class TestSystemMetricsPublisher {
timelineServer.start();
store = timelineServer.getTimelineStore();
- metricsPublisher = new SystemMetricsPublisher();
+ metricsPublisher = new SystemMetricsPublisher(mock(RMContext.class));
metricsPublisher.init(conf);
metricsPublisher.start();
}
[20/50] [abbrv] hadoop git commit: YARN-3047. [Data Serving] Set up
ATS reader with basic request serving structure and lifecycle (Varun Saxena
via sjlee)
Posted by sj...@apache.org.
YARN-3047. [Data Serving] Set up ATS reader with basic request serving structure and lifecycle (Varun Saxena via sjlee)
(cherry picked from commit 4c5f88fb0f04b7919738d07598b0f006a9ff91f2)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f0e7827
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f0e7827
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f0e7827
Branch: refs/heads/YARN-2928-rebase
Commit: 6f0e78278171a1cf070e4a3da2c321e5fe9474e5
Parents: 6d02412
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed Jul 8 17:10:10 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:38 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 5 +
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 8 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 3 +
.../hadoop/yarn/webapp/util/WebAppUtils.java | 6 +-
.../reader/TimelineReaderManager.java | 36 ++++
.../reader/TimelineReaderServer.java | 169 +++++++++++++++++++
.../reader/TimelineReaderWebServices.java | 59 +++++++
.../reader/TestTimelineReaderServer.java | 51 ++++++
.../reader/TestTimelineReaderWebServices.java | 121 +++++++++++++
10 files changed, 459 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f0e7827/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0d93cf8..e391cf9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -76,6 +76,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3051. Created storage oriented reader interface for fetching raw entity
data and made the filesystem based implementation. (Varun Saxena via zjshen)
+ YARN-3047. [Data Serving] Set up ATS reader with basic request serving
+ structure and lifecycle (Varun Saxena via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f0e7827/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index f0bed9b..b6d1204 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -43,6 +43,7 @@ function hadoop_usage
hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelineserver" "run the timeline server"
+ hadoop_add_subcommand "timelinereader" "run the timeline reader server"
hadoop_add_subcommand "top" "view cluster information"
hadoop_add_subcommand "version" "print the version"
hadoop_generate_usage "${MYNAME}" true
@@ -179,6 +180,10 @@ case "${COMMAND}" in
HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}"
fi
;;
+ timelinereader)
+ supportdaemonization="true"
+ CLASS='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
+ ;;
version)
CLASS=org.apache.hadoop.util.VersionInfo
hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f0e7827/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index 91c90fb..4c36307 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -151,7 +151,7 @@ if "%1" == "--loglevel" (
set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^
application applicationattempt container node queue logs daemonlog historyserver ^
- timelineserver classpath
+ timelineserver timelinereader classpath
for %%i in ( %yarncommands% ) do (
if %yarn-command% == %%i set yarncommand=true
)
@@ -242,6 +242,11 @@ goto :eof
)
goto :eof
+:timelinereader
+ set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties
+ set CLASS=org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer
+ goto :eof
+
:nodemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager
@@ -312,6 +317,7 @@ goto :eof
@echo resourcemanager run the ResourceManager
@echo nodemanager run a nodemanager on each slave
@echo timelineserver run the timeline server
+ @echo timelinereader run the timeline reader server
@echo rmadmin admin tools
@echo version print the version
@echo jar ^<jar^> run a jar file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f0e7827/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 524203b..750d426 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1513,6 +1513,9 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_SERVICE_WRITER_CLASS =
TIMELINE_SERVICE_PREFIX + "writer.class";
+ public static final String TIMELINE_SERVICE_READER_CLASS =
+ TIMELINE_SERVICE_PREFIX + "reader.class";
+
// mark app-history related configs @Private as application history is going
// to be integrated into the timeline service
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f0e7827/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index 8597427..f4a7416 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -263,6 +263,10 @@ public class WebAppUtils {
}
public static String getAHSWebAppURLWithoutScheme(Configuration conf) {
+ return getTimelineReaderWebAppURL(conf);
+ }
+
+ public static String getTimelineReaderWebAppURL(Configuration conf) {
if (YarnConfiguration.useHttps(conf)) {
return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS);
@@ -271,7 +275,7 @@ public class WebAppUtils {
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS);
}
}
-
+
/**
* if url has scheme then it will be returned as it is else it will return
* url with scheme.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f0e7827/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
new file mode 100644
index 0000000..5573185
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+
+@Private
+@Unstable
+public class TimelineReaderManager extends AbstractService {
+
+ private TimelineReader reader;
+
+ public TimelineReaderManager(TimelineReader timelineReader) {
+ super(TimelineReaderManager.class.getName());
+ this.reader = timelineReader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f0e7827/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
new file mode 100644
index 0000000..874112c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.http.lib.StaticUserWebFilter;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/** Main class for Timeline Reader */
+@Private
+@Unstable
+public class TimelineReaderServer extends CompositeService {
+ private static final Log LOG = LogFactory.getLog(TimelineReaderServer.class);
+ private static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private static final String TIMELINE_READER_MANAGER_ATTR =
+ "timeline.reader.manager";
+
+ private HttpServer2 readerWebServer;
+ private TimelineReaderManager timelineReaderManager;
+
+ public TimelineReaderServer() {
+ super(TimelineReaderServer.class.getName());
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ TimelineReader timelineReaderStore = createTimelineReaderStore(conf);
+ addService(timelineReaderStore);
+ timelineReaderManager = createTimelineReaderManager(timelineReaderStore);
+ addService(timelineReaderManager);
+ super.serviceInit(conf);
+ }
+
+ private TimelineReader createTimelineReaderStore(Configuration conf) {
+ TimelineReader readerStore = ReflectionUtils.newInstance(conf.getClass(
+ YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
+ FileSystemTimelineReaderImpl.class, TimelineReader.class), conf);
+ readerStore.init(conf);
+ return readerStore;
+ }
+
+ private TimelineReaderManager createTimelineReaderManager(
+ TimelineReader timelineReaderStore) {
+ return new TimelineReaderManager(timelineReaderStore);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ startTimelineReaderWebApp();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (readerWebServer != null) {
+ readerWebServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ private void startTimelineReaderWebApp() {
+ Configuration conf = getConfig();
+ String bindAddress = WebAppUtils.getWebAppBindURL(conf,
+ YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ WebAppUtils.getTimelineReaderWebAppURL(conf));
+ LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress);
+ try {
+ HttpServer2.Builder builder = new HttpServer2.Builder()
+ .setName("timeline")
+ .setConf(conf)
+ .addEndpoint(URI.create("http://" + bindAddress));
+ readerWebServer = builder.build();
+
+ Map<String, String> options = new HashMap<>();
+ String username = conf.get(HADOOP_HTTP_STATIC_USER,
+ DEFAULT_HADOOP_HTTP_STATIC_USER);
+ options.put(HADOOP_HTTP_STATIC_USER, username);
+ HttpServer2.defineFilter(readerWebServer.getWebAppContext(),
+ "static_user_filter_timeline",
+ StaticUserWebFilter.StaticUserFilter.class.getName(),
+ options, new String[] {"/*"});
+
+ readerWebServer.addJerseyResourcePackage(
+ TimelineReaderWebServices.class.getPackage().getName() + ";"
+ + GenericExceptionHandler.class.getPackage().getName() + ";"
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName(),
+ "/*");
+ readerWebServer.setAttribute(TIMELINE_READER_MANAGER_ATTR,
+ timelineReaderManager);
+ readerWebServer.start();
+ } catch (Exception e) {
+ String msg = "TimelineReaderWebApp failed to start.";
+ LOG.error(msg, e);
+ throw new YarnRuntimeException(msg, e);
+ }
+ }
+
+ @VisibleForTesting
+ int getWebServerPort() {
+ return readerWebServer.getConnectorAddress(0).getPort();
+ }
+
+ static TimelineReaderServer startTimelineReaderServer(String[] args) {
+ Thread.setDefaultUncaughtExceptionHandler(
+ new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(TimelineReaderServer.class,
+ args, LOG);
+ TimelineReaderServer timelineReaderServer = null;
+ try {
+ timelineReaderServer = new TimelineReaderServer();
+ ShutdownHookManager.get().addShutdownHook(
+ new CompositeServiceShutdownHook(timelineReaderServer),
+ SHUTDOWN_HOOK_PRIORITY);
+ YarnConfiguration conf = new YarnConfiguration();
+ timelineReaderServer.init(conf);
+ timelineReaderServer.start();
+ } catch (Throwable t) {
+ LOG.fatal("Error starting TimelineReaderWebServer", t);
+ ExitUtil.terminate(-1, "Error starting TimelineReaderWebServer");
+ }
+ return timelineReaderServer;
+ }
+
+ public static void main(String[] args) {
+ startTimelineReaderServer(args);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f0e7827/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
new file mode 100644
index 0000000..3655a72
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+import com.google.inject.Singleton;
+
+
+/** REST end point for Timeline Reader */
+@Private
+@Unstable
+@Singleton
+@Path("/ws/v2/timeline")
+public class TimelineReaderWebServices {
+
+ private void init(HttpServletResponse response) {
+ response.setContentType(null);
+ }
+
+ /**
+ * Return the description of the timeline reader web services.
+ */
+ @GET
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public TimelineAbout about(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res) {
+ init(res);
+ return TimelineUtils.createTimelineAbout("Timeline Reader API");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f0e7827/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
new file mode 100644
index 0000000..7098814
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer;
+import org.junit.Test;
+
+public class TestTimelineReaderServer {
+
+ @Test(timeout = 60000)
+ public void testStartStopServer() throws Exception {
+ TimelineReaderServer server = new TimelineReaderServer();
+ Configuration config = new YarnConfiguration();
+ config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ "localhost:0");
+ try {
+ server.init(config);
+ assertEquals(STATE.INITED, server.getServiceState());
+ assertEquals(2, server.getServices().size());
+
+ server.start();
+ assertEquals(STATE.STARTED, server.getServiceState());
+
+ server.stop();
+ assertEquals(STATE.STOPPED, server.getServiceState());
+ } finally {
+ server.stop();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f0e7827/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
new file mode 100644
index 0000000..a9145d0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+
+import javax.ws.rs.core.MediaType;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+
+public class TestTimelineReaderWebServices {
+ private int serverPort;
+ private TimelineReaderServer server;
+
+ @Before
+ public void init() throws Exception {
+ try {
+ Configuration config = new YarnConfiguration();
+ config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ "localhost:0");
+ server = new TimelineReaderServer();
+ server.init(config);
+ server.start();
+ serverPort = server.getWebServerPort();
+ } catch (Exception e) {
+ Assert.fail("Web server failed to start");
+ }
+ }
+
+ @After
+ public void stop() throws Exception {
+ if (server != null) {
+ server.stop();
+ server = null;
+ }
+ }
+
+ private static Client createClient() {
+ ClientConfig cfg = new DefaultClientConfig();
+ cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
+ return new Client(new URLConnectionClientHandler(
+ new DummyURLConnectionFactory()), cfg);
+ }
+
+ private static ClientResponse getResponse(Client client, URI uri) throws Exception {
+ ClientResponse resp =
+ client.resource(uri).accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ if (resp == null ||
+ resp.getClientResponseStatus() != ClientResponse.Status.OK) {
+ System.out.println(resp.getClientResponseStatus());
+ throw new IOException("Incorrect response from timeline reader.");
+ }
+ return resp;
+ }
+
+ private static class DummyURLConnectionFactory
+ implements HttpURLConnectionFactory {
+
+ @Override
+ public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
+ try {
+ return (HttpURLConnection)url.openConnection();
+ } catch (UndeclaredThrowableException e) {
+ throw new IOException(e.getCause());
+ }
+ }
+ }
+
+ @Test
+ public void testAbout()
+ throws IOException {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/");
+ Client client = createClient();
+ try {
+ ClientResponse resp = getResponse(client, uri);
+ TimelineAbout about = resp.getEntity(TimelineAbout.class);
+ Assert.assertNotNull(about);
+ Assert.assertEquals("Timeline Reader API", about.getAbout());
+ } catch (Exception re) {
+ throw new IOException(
+ "Failed to get the response from timeline reader.", re);
+ } finally {
+ client.destroy();
+ }
+ }
+}
\ No newline at end of file
[44/50] [abbrv] hadoop git commit: YARN-4075 [reader REST API]
implement support for querying for flows and flow runs (Varun Saxena via
vrushali)
Posted by sj...@apache.org.
YARN-4075 [reader REST API] implement support for querying for flows and flow runs (Varun Saxena via vrushali)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea801ebf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea801ebf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea801ebf
Branch: refs/heads/YARN-2928-rebase
Commit: ea801ebf06d47350148568f8aee36ca66ecc9587
Parents: c4d8d8f
Author: Vrushali Channapattan <vr...@apache.org>
Authored: Fri Sep 25 12:16:38 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 17:05:04 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/timelineservice/TimelineMetric.java | 10 +
.../reader/TimelineReaderManager.java | 25 +-
.../reader/TimelineReaderServer.java | 1 +
.../reader/TimelineReaderWebServices.java | 243 ++++++++++--
.../timelineservice/reader/package-info.java | 23 ++
.../storage/TimelineSchemaCreator.java | 3 +
.../reader/TestTimelineReaderWebServices.java | 52 ++-
.../TestTimelineReaderWebServicesFlowRun.java | 365 +++++++++++++++++++
.../src/test/resources/log4j.properties | 19 +
10 files changed, 689 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea801ebf/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3d2de5c..867c2aa 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -109,6 +109,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-4074. [timeline reader] implement support for querying for flows
and flow runs (sjlee via vrushali)
+ YARN-4075. [reader REST API] implement support for querying for flows
+ and flow runs (Varun Saxena via vrushali)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea801ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index f44c1a2..e3870da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -154,4 +154,14 @@ public class TimelineMetric {
}
return true;
}
+
+ @Override
+ public String toString() {
+ String str = "{id:" + id + ", type:" + type;
+ if (!values.isEmpty()) {
+ str += ", values:" + values;
+ }
+ str += "}";
+ return str;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea801ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
index 7fafd82..27a50d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -25,9 +25,10 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
@@ -43,6 +44,22 @@ public class TimelineReaderManager extends AbstractService {
}
/**
+ * Gets cluster ID from config yarn.resourcemanager.cluster-id
+ * if not supplied by client.
+ * @param clusterId
+ * @param conf
+ * @return clusterId
+ */
+ private static String getClusterID(String clusterId, Configuration conf) {
+ if (clusterId == null || clusterId.isEmpty()) {
+ return conf.get(
+ YarnConfiguration.RM_CLUSTER_ID,
+ YarnConfiguration.DEFAULT_RM_CLUSTER_ID);
+ }
+ return clusterId;
+ }
+
+ /**
* Get a set of entities matching given predicates. The meaning of each
* argument has been documented with {@link TimelineReader#getEntities}.
*
@@ -56,7 +73,8 @@ public class TimelineReaderManager extends AbstractService {
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
EnumSet<Field> fieldsToRetrieve) throws IOException {
- return reader.getEntities(userId, clusterId, flowId, flowRunId, appId,
+ String cluster = getClusterID(clusterId, getConfig());
+ return reader.getEntities(userId, cluster, flowId, flowRunId, appId,
entityType, limit, createdTimeBegin, createdTimeEnd, modifiedTimeBegin,
modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters,
metricFilters, eventFilters, fieldsToRetrieve);
@@ -71,7 +89,8 @@ public class TimelineReaderManager extends AbstractService {
public TimelineEntity getEntity(String userId, String clusterId,
String flowId, Long flowRunId, String appId, String entityType,
String entityId, EnumSet<Field> fields) throws IOException {
- return reader.getEntity(userId, clusterId, flowId, flowRunId, appId,
+ String cluster = getClusterID(clusterId, getConfig());
+ return reader.getEntity(userId, cluster, flowId, flowRunId, appId,
entityType, entityId, fields);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea801ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
index 319cfb0..afe1536 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -77,6 +77,7 @@ public class TimelineReaderServer extends CompositeService {
TimelineReader readerStore = ReflectionUtils.newInstance(conf.getClass(
YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
FileSystemTimelineReaderImpl.class, TimelineReader.class), conf);
+ LOG.info("Using store " + readerStore.getClass().getName());
readerStore.init(conf);
return readerStore;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea801ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index 0b5fde0..f619c7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
import java.io.IOException;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -45,6 +46,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
@@ -169,6 +171,11 @@ public class TimelineReaderWebServices {
return str == null ? null : str.trim();
}
+ private static String parseUser(UserGroupInformation callerUGI, String user) {
+ return (callerUGI != null && (user == null || user.isEmpty()) ?
+ callerUGI.getUserName().trim() : parseStr(user));
+ }
+
private static UserGroupInformation getUser(HttpServletRequest req) {
String remoteUser = req.getRemoteUser();
UserGroupInformation callerUGI = null;
@@ -183,6 +190,17 @@ public class TimelineReaderWebServices {
ctxt.getAttribute(TimelineReaderServer.TIMELINE_READER_MANAGER_ATTR);
}
+ private static void handleException(Exception e) throws BadRequestException,
+ WebApplicationException {
+ if (e instanceof IllegalArgumentException) {
+ throw new BadRequestException("Requested Invalid Field.");
+ } else {
+ LOG.error("Error while processing REST request", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
/**
* Return the description of the timeline reader web services.
*/
@@ -196,25 +214,58 @@ public class TimelineReaderWebServices {
}
/**
+ * Return a set of entities that match the given parameters. Cluster ID is not
+ * provided by client so default cluster ID has to be taken.
+ */
+ @GET
+ @Path("/entities/{appid}/{entitytype}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getEntities(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("appid") String appId,
+ @PathParam("entitytype") String entityType,
+ @QueryParam("userid") String userId,
+ @QueryParam("flowid") String flowId,
+ @QueryParam("flowrunid") String flowRunId,
+ @QueryParam("limit") String limit,
+ @QueryParam("createdtimestart") String createdTimeStart,
+ @QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("modifiedtimestart") String modifiedTimeStart,
+ @QueryParam("modifiedtimeend") String modifiedTimeEnd,
+ @QueryParam("relatesto") String relatesTo,
+ @QueryParam("isrelatedto") String isRelatedTo,
+ @QueryParam("infofilters") String infofilters,
+ @QueryParam("conffilters") String conffilters,
+ @QueryParam("metricfilters") String metricfilters,
+ @QueryParam("eventfilters") String eventfilters,
+ @QueryParam("fields") String fields) {
+ return getEntities(req, res, null, appId, entityType, userId, flowId,
+ flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
+ modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
+ metricfilters, eventfilters, fields);
+ }
+
+ /**
* Return a set of entities that match the given parameters.
*/
@GET
- @Path("/entities/{clusterId}/{appId}/{entityType}")
+ @Path("/entities/{clusterid}/{appid}/{entitytype}/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getEntities(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
- @PathParam("clusterId") String clusterId,
- @PathParam("appId") String appId,
- @PathParam("entityType") String entityType,
- @QueryParam("userId") String userId,
- @QueryParam("flowId") String flowId,
- @QueryParam("flowRunId") String flowRunId,
+ @PathParam("clusterid") String clusterId,
+ @PathParam("appid") String appId,
+ @PathParam("entitytype") String entityType,
+ @QueryParam("userid") String userId,
+ @QueryParam("flowid") String flowId,
+ @QueryParam("flowrunid") String flowRunId,
@QueryParam("limit") String limit,
- @QueryParam("createdTimeStart") String createdTimeStart,
- @QueryParam("createdTimeEnd") String createdTimeEnd,
- @QueryParam("modifiedTimeStart") String modifiedTimeStart,
- @QueryParam("modifiedTimeEnd") String modifiedTimeEnd,
+ @QueryParam("createdtimestart") String createdTimeStart,
+ @QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("modifiedtimestart") String modifiedTimeStart,
+ @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -225,11 +276,10 @@ public class TimelineReaderWebServices {
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
UserGroupInformation callerUGI = getUser(req);
+ Set<TimelineEntity> entities = null;
try {
- return timelineReaderManager.getEntities(
- callerUGI != null && (userId == null || userId.isEmpty()) ?
- callerUGI.getUserName().trim() : parseStr(userId),
- parseStr(clusterId), parseStr(flowId),
+ entities = timelineReaderManager.getEntities(
+ parseUser(callerUGI, userId), parseStr(clusterId), parseStr(flowId),
parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
parseLongStr(limit), parseLongStr(createdTimeStart),
parseLongStr(createdTimeEnd), parseLongStr(modifiedTimeStart),
@@ -245,31 +295,52 @@ public class TimelineReaderWebServices {
throw new BadRequestException(
"createdTime or modifiedTime start/end or limit or flowId is not" +
" a numeric value.");
- } catch (IllegalArgumentException e) {
- throw new BadRequestException("Requested Invalid Field.");
} catch (Exception e) {
- LOG.error("Error getting entities", e);
- throw new WebApplicationException(e,
- Response.Status.INTERNAL_SERVER_ERROR);
+ handleException(e);
+ }
+ if (entities == null) {
+ entities = Collections.emptySet();
}
+ return entities;
+ }
+
+ /**
+ * Return a single entity of the given entity type and Id. Cluster ID is not
+ * provided by client so default cluster ID has to be taken.
+ */
+ @GET
+ @Path("/entity/{appid}/{entitytype}/{entityid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public TimelineEntity getEntity(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("appid") String appId,
+ @PathParam("entitytype") String entityType,
+ @PathParam("entityid") String entityId,
+ @QueryParam("userid") String userId,
+ @QueryParam("flowid") String flowId,
+ @QueryParam("flowrunid") String flowRunId,
+ @QueryParam("fields") String fields) {
+ return getEntity(req, res, null, appId, entityType, entityId, userId,
+ flowId, flowRunId, fields);
}
/**
* Return a single entity of the given entity type and Id.
*/
@GET
- @Path("/entity/{clusterId}/{appId}/{entityType}/{entityId}/")
+ @Path("/entity/{clusterid}/{appid}/{entitytype}/{entityid}/")
@Produces(MediaType.APPLICATION_JSON)
public TimelineEntity getEntity(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
- @PathParam("clusterId") String clusterId,
- @PathParam("appId") String appId,
- @PathParam("entityType") String entityType,
- @PathParam("entityId") String entityId,
- @QueryParam("userId") String userId,
- @QueryParam("flowId") String flowId,
- @QueryParam("flowRunId") String flowRunId,
+ @PathParam("clusterid") String clusterId,
+ @PathParam("appid") String appId,
+ @PathParam("entitytype") String entityType,
+ @PathParam("entityid") String entityId,
+ @QueryParam("userid") String userId,
+ @QueryParam("flowid") String flowId,
+ @QueryParam("flowrunid") String flowRunId,
@QueryParam("fields") String fields) {
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
@@ -277,19 +348,13 @@ public class TimelineReaderWebServices {
TimelineEntity entity = null;
try {
entity = timelineReaderManager.getEntity(
- callerUGI != null && (userId == null || userId.isEmpty()) ?
- callerUGI.getUserName().trim() : parseStr(userId),
- parseStr(clusterId), parseStr(flowId), parseLongStr(flowRunId),
- parseStr(appId), parseStr(entityType), parseStr(entityId),
- parseFieldsStr(fields, COMMA_DELIMITER));
+ parseUser(callerUGI, userId), parseStr(clusterId), parseStr(flowId),
+ parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
+ parseStr(entityId), parseFieldsStr(fields, COMMA_DELIMITER));
} catch (NumberFormatException e) {
- throw new BadRequestException("flowRunId is not a numeric value.");
- } catch (IllegalArgumentException e) {
- throw new BadRequestException("Requested Invalid Field.");
+ throw new BadRequestException("flowrunid is not a numeric value.");
} catch (Exception e) {
- LOG.error("Error getting entity", e);
- throw new WebApplicationException(e,
- Response.Status.INTERNAL_SERVER_ERROR);
+ handleException(e);
}
if (entity == null) {
throw new NotFoundException("Timeline entity {id: " + parseStr(entityId) +
@@ -297,4 +362,104 @@ public class TimelineReaderWebServices {
}
return entity;
}
+
+ /**
+ * Return a single flow run for the given cluster, flow id and run id.
+ * Cluster ID is not provided by client so default cluster ID has to be taken.
+ */
+ @GET
+ @Path("/flowrun/{flowid}/{flowrunid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public TimelineEntity getFlowRun(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("flowid") String flowId,
+ @PathParam("flowrunid") String flowRunId,
+ @QueryParam("userid") String userId,
+ @QueryParam("fields") String fields) {
+ return getFlowRun(req, res, null, flowId, flowRunId, userId, fields);
+ }
+
+ /**
+ * Return a single flow run for the given cluster, flow id and run id.
+ */
+ @GET
+ @Path("/flowrun/{clusterid}/{flowid}/{flowrunid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public TimelineEntity getFlowRun(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("clusterid") String clusterId,
+ @PathParam("flowid") String flowId,
+ @PathParam("flowrunid") String flowRunId,
+ @QueryParam("userid") String userId,
+ @QueryParam("fields") String fields) {
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ UserGroupInformation callerUGI = getUser(req);
+ TimelineEntity entity = null;
+ try {
+ entity = timelineReaderManager.getEntity(
+ parseUser(callerUGI, userId), parseStr(clusterId),
+ parseStr(flowId), parseLongStr(flowRunId), null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null,
+ parseFieldsStr(fields, COMMA_DELIMITER));
+ } catch (NumberFormatException e) {
+ throw new BadRequestException("flowRunId is not a numeric value.");
+ } catch (Exception e) {
+ handleException(e);
+ }
+ if (entity == null) {
+ throw new NotFoundException("Flow run {flow id: " + parseStr(flowId) +
+ ", run id: " + parseLongStr(flowRunId) + " } is not found");
+ }
+ return entity;
+ }
+
+ /**
+ * Return a list of flows for a given cluster id. Cluster ID is not
+ * provided by client so default cluster ID has to be taken.
+ */
+ @GET
+ @Path("/flows/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getFlows(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("limit") String limit,
+ @QueryParam("fields") String fields) {
+ return getFlows(req, res, null, limit, fields);
+ }
+
+ /**
+ * Return a list of flows for a given cluster id.
+ */
+ @GET
+ @Path("/flows/{clusterid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getFlows(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("clusterid") String clusterId,
+ @QueryParam("limit") String limit,
+ @QueryParam("fields") String fields) {
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ Set<TimelineEntity> entities = null;
+ try {
+ entities = timelineReaderManager.getEntities(
+ null, parseStr(clusterId), null, null, null,
+ TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), parseLongStr(limit),
+ null, null, null, null, null, null, null, null, null, null,
+ parseFieldsStr(fields, COMMA_DELIMITER));
+ } catch (NumberFormatException e) {
+ throw new BadRequestException("limit is not a numeric value.");
+ } catch (Exception e) {
+ handleException(e);
+ }
+ if (entities == null) {
+ entities = Collections.emptySet();
+ }
+ return entities;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea801ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java
new file mode 100644
index 0000000..51247bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea801ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index cbcff4c..46bc2e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* This creates the schema for a hbase based backend for storing application
* timeline information.
@@ -201,6 +203,7 @@ public class TimelineSchemaCreator {
return commandLine;
}
+ @VisibleForTesting
public static void createAllTables(Configuration hbaseConf,
boolean skipExisting) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea801ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index 0f7c22f..45bce2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -187,8 +187,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entity/cluster1/app1/app/id_1?userId=user1&" +
- "flowId=flow1&flowRunId=1");
+ "timeline/entity/cluster1/app1/app/id_1?userid=user1&" +
+ "flowid=flow1&flowrunid=1");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -259,6 +259,32 @@ public class TestTimelineReaderWebServices {
}
@Test
+ public void testQueryWithoutCluster() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/app1/app/id_1");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("id_1", entity.getId());
+ assertEquals("app", entity.getType());
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/app1/app");
+ resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(4, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
public void testGetEntities() throws Exception {
Client client = createClient();
try {
@@ -318,8 +344,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?createdTimeStart=1425016502030&"
- + "createdTimeEnd=1425016502060");
+ "timeline/entities/cluster1/app1/app?createdtimestart=1425016502030&"
+ + "createdtimeend=1425016502060");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -330,7 +356,7 @@ public class TestTimelineReaderWebServices {
entities.contains(newEntity("app", "id_4")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?createdTimeEnd=1425016502010");
+ "entities/cluster1/app1/app?createdtimeend=1425016502010");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -340,7 +366,7 @@ public class TestTimelineReaderWebServices {
entities.contains(newEntity("app", "id_4")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?createdTimeStart=1425016502010");
+ "entities/cluster1/app1/app?createdtimestart=1425016502010");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -358,8 +384,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?modifiedTimeStart=1425016502090"
- + "&modifiedTimeEnd=1425016503020");
+ "timeline/entities/cluster1/app1/app?modifiedtimestart=1425016502090"
+ + "&modifiedtimeend=1425016503020");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -372,7 +398,7 @@ public class TestTimelineReaderWebServices {
entities.contains(newEntity("app", "id_4")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?modifiedTimeEnd=1425016502090");
+ "entities/cluster1/app1/app?modifiedtimeend=1425016502090");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -384,7 +410,7 @@ public class TestTimelineReaderWebServices {
entities.contains(newEntity("app", "id_3")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?modifiedTimeStart=1425016503005");
+ "entities/cluster1/app1/app?modifiedtimestart=1425016503005");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -527,7 +553,7 @@ public class TestTimelineReaderWebServices {
"timeline/entities/cluster1/app1/app?metricfilters=metric7&" +
"isrelatedto=type1:tid1_1;tid1_2,type2:tid2_1%60&relatesto=" +
"flow:flow1&eventfilters=event_2,event_4&infofilters=info2:3.5" +
- "&createdTimeStart=1425016502030&createdTimeEnd=1425016502060");
+ "&createdtimestart=1425016502030&createdtimeend=1425016502060");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -544,11 +570,11 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?flowRunId=a23b");
+ "timeline/entities/cluster1/app1/app?flowrunid=a23b");
verifyHttpResponse(client, uri, Status.BAD_REQUEST);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entity/cluster1/app1/app/id_1?flowRunId=2ab15");
+ "entity/cluster1/app1/app/id_1?flowrunid=2ab15");
verifyHttpResponse(client, uri, Status.BAD_REQUEST);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea801ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java
new file mode 100644
index 0000000..ae71e2c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java
@@ -0,0 +1,365 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.core.MediaType;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+
+public class TestTimelineReaderWebServicesFlowRun {
+ private int serverPort;
+ private TimelineReaderServer server;
+ private static HBaseTestingUtility util;
+ private static long ts = System.currentTimeMillis();
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ util = new HBaseTestingUtility();
+ Configuration conf = util.getConfiguration();
+ conf.setInt("hfile.format.version", 3);
+ util.startMiniCluster();
+ TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
+ loadData();
+ }
+
+ private static void loadData() throws Exception {
+ String cluster = "cluster1";
+ String user = "user1";
+ String flow = "flow_name";
+ String flowVersion = "CF7022C10F1354";
+ Long runid = 1002345678919L;
+ Long runid1 = 1002345678920L;
+
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowRunMetrics_test";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ Long cTime = 1425016501000L;
+ entity.setCreatedTime(cTime);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ metricValues.put(ts - 100000, 2);
+ metricValues.put(ts - 80000, 40);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+
+ m1 = new TimelineMetric();
+ m1.setId("HDFS_BYTES_READ");
+ metricValues = new HashMap<Long, Number>();
+ metricValues.put(ts - 100000, 31);
+ metricValues.put(ts - 80000, 57);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ Long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+ String expKey = "foo_event";
+ Object expVal = "test";
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
+ te.addEntity(entity);
+
+ // write another application with same metric to this flow
+ TimelineEntities te1 = new TimelineEntities();
+ TimelineEntity entity1 = new TimelineEntity();
+ id = "flowRunMetrics_test";
+ type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity1.setId(id);
+ entity1.setType(type);
+ cTime = 1425016501000L;
+ entity1.setCreatedTime(cTime);
+ // add metrics
+ metrics.clear();
+ TimelineMetric m2 = new TimelineMetric();
+ m2.setId("MAP_SLOT_MILLIS");
+ metricValues = new HashMap<Long, Number>();
+ metricValues.put(ts - 100000, 5L);
+ metricValues.put(ts - 80000, 101L);
+ m2.setType(Type.TIME_SERIES);
+ m2.setValues(metricValues);
+ metrics.add(m2);
+ entity1.addMetrics(metrics);
+ te1.addEntity(entity1);
+
+ String flow2 = "flow_name2";
+ String flowVersion2 = "CF7022C10F1454";
+ Long runid2 = 2102356789046L;
+ TimelineEntities te3 = new TimelineEntities();
+ TimelineEntity entity3 = new TimelineEntity();
+ id = "flowRunMetrics_test1";
+ entity3.setId(id);
+ entity3.setType(type);
+ cTime = 1425016501030L;
+ entity3.setCreatedTime(cTime);
+ TimelineEvent event2 = new TimelineEvent();
+ event2.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ event2.setTimestamp(1436512802030L);
+ event2.addInfo("foo_event", "test");
+ entity3.addEvent(event2);
+ te3.addEntity(entity3);
+
+ HBaseTimelineWriterImpl hbi = null;
+ Configuration c1 = util.getConfiguration();
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ String appName = "application_11111111111111_1111";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+ appName = "application_11111111111111_2222";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
+ hbi.write(cluster, user, flow, flowVersion, runid1, appName, te);
+ appName = "application_11111111111111_2223";
+ hbi.write(cluster, user, flow2, flowVersion2, runid2, appName, te3);
+ hbi.flush();
+ } finally {
+ hbi.close();
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ util.shutdownMiniCluster();
+ }
+
+ @Before
+ public void init() throws Exception {
+ try {
+ Configuration config = util.getConfiguration();
+ config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ "localhost:0");
+ config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
+ config.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
+ "org.apache.hadoop.yarn.server.timelineservice.storage." +
+ "HBaseTimelineReaderImpl");
+ config.setInt("hfile.format.version", 3);
+ server = new TimelineReaderServer();
+ server.init(config);
+ server.start();
+ serverPort = server.getWebServerPort();
+ } catch (Exception e) {
+ Assert.fail("Web server failed to start");
+ }
+ }
+
+ private static Client createClient() {
+ ClientConfig cfg = new DefaultClientConfig();
+ cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
+ return new Client(new URLConnectionClientHandler(
+ new DummyURLConnectionFactory()), cfg);
+ }
+
+ private static ClientResponse getResponse(Client client, URI uri)
+ throws Exception {
+ ClientResponse resp =
+ client.resource(uri).accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ if (resp == null ||
+ resp.getClientResponseStatus() != ClientResponse.Status.OK) {
+ String msg = new String();
+ if (resp != null) {
+ msg = resp.getClientResponseStatus().toString();
+ }
+ throw new IOException("Incorrect response from timeline reader. " +
+ "Status=" + msg);
+ }
+ return resp;
+ }
+
+ private static class DummyURLConnectionFactory
+ implements HttpURLConnectionFactory {
+
+ @Override
+ public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
+ try {
+ return (HttpURLConnection)url.openConnection();
+ } catch (UndeclaredThrowableException e) {
+ throw new IOException(e.getCause());
+ }
+ }
+ }
+
+ private static TimelineMetric newMetric(String id, long ts, Number value) {
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId(id);
+ metric.addValue(ts, value);
+ return metric;
+ }
+
+ private static boolean verifyMetricValues(Map<Long, Number> m1,
+ Map<Long, Number> m2) {
+ for (Map.Entry<Long, Number> entry : m1.entrySet()) {
+ if (!m2.containsKey(entry.getKey())) {
+ return false;
+ }
+ if (m2.get(entry.getKey()).equals(entry.getValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean verifyMetrics(
+ TimelineMetric m, TimelineMetric... metrics) {
+ for (TimelineMetric metric : metrics) {
+ if (!metric.equals(m)) {
+ continue;
+ }
+ if (!verifyMetricValues(metric.getValues(), m.getValues())) {
+ continue;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Test
+ public void testGetFlowRun() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowrun/cluster1/flow_name/1002345678919?userid=user1");
+ ClientResponse resp = getResponse(client, uri);
+ FlowRunEntity entity = resp.getEntity(FlowRunEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("user1@flow_name/1002345678919", entity.getId());
+ assertEquals(2, entity.getMetrics().size());
+ TimelineMetric m1 = newMetric("HDFS_BYTES_READ", ts - 80000, 57L);
+ TimelineMetric m2 = newMetric("MAP_SLOT_MILLIS", ts - 80000, 141L);
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(verifyMetrics(metric, m1, m2));
+ }
+
+ // Query without specifying cluster ID.
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flowrun/flow_name/1002345678919?userid=user1");
+ resp = getResponse(client, uri);
+ entity = resp.getEntity(FlowRunEntity.class);
+ assertNotNull(entity);
+ assertEquals("user1@flow_name/1002345678919", entity.getId());
+ assertEquals(2, entity.getMetrics().size());
+ m1 = newMetric("HDFS_BYTES_READ", ts - 80000, 57L);
+ m2 = newMetric("MAP_SLOT_MILLIS", ts - 80000, 141L);
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(verifyMetrics(metric, m1, m2));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetFlows() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flows/cluster1");
+ ClientResponse resp = getResponse(client, uri);
+ Set<FlowActivityEntity> entities =
+ resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (FlowActivityEntity entity : entities) {
+ assertTrue((entity.getId().endsWith("@flow_name") &&
+ entity.getFlowRuns().size() == 2) ||
+ (entity.getId().endsWith("@flow_name2") &&
+ entity.getFlowRuns().size() == 1));
+ }
+
+ // Query without specifying cluster ID.
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flows/");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flows/cluster1?limit=1");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @After
+ public void stop() throws Exception {
+ if (server != null) {
+ server.stop();
+ server = null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea801ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/resources/log4j.properties
new file mode 100644
index 0000000..81a3f6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
[23/50] [abbrv] hadoop git commit: YARN-3949. Ensure timely flush of
timeline writes. Contributed by Sangjin Lee.
Posted by sj...@apache.org.
YARN-3949. Ensure timely flush of timeline writes. Contributed by Sangjin Lee.
(cherry picked from commit 967bef7e0396d857913caa2574afb103a5f0b81b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b2cff69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b2cff69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b2cff69
Branch: refs/heads/YARN-2928-rebase
Commit: 0b2cff693c6cd8bcb5a5df91253407b08aea2a4b
Parents: 95e7607
Author: Junping Du <ju...@apache.org>
Authored: Sat Jul 25 10:30:29 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Sat Oct 10 15:45:39 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 9 +++
.../src/main/resources/yarn-default.xml | 17 ++++-
.../collector/TimelineCollectorManager.java | 65 ++++++++++++++++++--
.../storage/FileSystemTimelineWriterImpl.java | 5 ++
.../storage/HBaseTimelineWriterImpl.java | 6 ++
.../storage/PhoenixTimelineWriterImpl.java | 5 ++
.../timelineservice/storage/TimelineWriter.java | 9 +++
.../TestNMTimelineCollectorManager.java | 5 ++
9 files changed, 119 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b2cff69/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8542191..564ca5a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -79,6 +79,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3047. [Data Serving] Set up ATS reader with basic request serving
structure and lifecycle (Varun Saxena via sjlee)
+ YARN-3949. Ensure timely flush of timeline writes. (Sangjin Lee via
+ junping_du)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b2cff69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 750d426..e2e0635 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1516,6 +1516,15 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_SERVICE_READER_CLASS =
TIMELINE_SERVICE_PREFIX + "reader.class";
+ /** The setting that controls how often the timeline collector flushes the
+ * timeline writer.
+ */
+ public static final String TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS =
+ TIMELINE_SERVICE_PREFIX + "writer.flush-interval-seconds";
+
+ public static final int
+ DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS = 60;
+
// mark app-history related configs @Private as application history is going
// to be integrated into the timeline service
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b2cff69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 54462e2..8626487 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -767,7 +767,15 @@
<name>yarn.system-metrics-publisher.enabled</name>
<value>false</value>
</property>
-
+
+ <property>
+ <description>The setting that controls whether yarn container metrics is
+ published to the timeline server or not by RM. This configuration setting is
+ for ATS V2.</description>
+ <name>yarn.rm.system-metrics-publisher.emit-container-events</name>
+ <value>false</value>
+ </property>
+
<property>
<description>Number of worker threads that send the yarn system metrics
@@ -1908,6 +1916,13 @@
<value>${hadoop.tmp.dir}/yarn/timeline</value>
</property>
+ <property>
+ <description>The setting that controls how often the timeline collector
+ flushes the timeline writer.</description>
+ <name>yarn.timeline-service.writer.flush-interval-seconds</name>
+ <value>60</value>
+ </property>
+
<!-- Shared Cache Configuration -->
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b2cff69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 23ad4f4..e9f2085 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -19,6 +19,13 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -32,9 +39,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
+import com.google.common.annotations.VisibleForTesting;
/**
* Class that manages adding and removing collectors and their lifecycle. It
@@ -47,7 +52,10 @@ public abstract class TimelineCollectorManager extends AbstractService {
private static final Log LOG =
LogFactory.getLog(TimelineCollectorManager.class);
- protected TimelineWriter writer;
+ private TimelineWriter writer;
+ private ScheduledExecutorService writerFlusher;
+ private int flushInterval;
+ private boolean writerFlusherRunning;
@Override
public void serviceInit(Configuration conf) throws Exception {
@@ -56,6 +64,12 @@ public abstract class TimelineCollectorManager extends AbstractService {
FileSystemTimelineWriterImpl.class,
TimelineWriter.class), conf);
writer.init(conf);
+ // create a single dedicated thread for flushing the writer on a periodic
+ // basis
+ writerFlusher = Executors.newSingleThreadScheduledExecutor();
+ flushInterval = conf.getInt(
+ YarnConfiguration.TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS);
super.serviceInit(conf);
}
@@ -65,6 +79,10 @@ public abstract class TimelineCollectorManager extends AbstractService {
if (writer != null) {
writer.start();
}
+ // schedule the flush task
+ writerFlusher.scheduleAtFixedRate(new WriterFlushTask(writer),
+ flushInterval, flushInterval, TimeUnit.SECONDS);
+ writerFlusherRunning = true;
}
// access to this map is synchronized with the map itself
@@ -161,9 +179,48 @@ public abstract class TimelineCollectorManager extends AbstractService {
c.serviceStop();
}
}
+ // stop the flusher first
+ if (writerFlusher != null) {
+ writerFlusher.shutdown();
+ writerFlusherRunning = false;
+ if (!writerFlusher.awaitTermination(30, TimeUnit.SECONDS)) {
+ // in reality it should be ample time for the flusher task to finish
+ // even if it times out, writers may be able to handle closing in this
+ // situation fine
+ // proceed to close the writer
+ LOG.warn("failed to stop the flusher task in time. " +
+ "will still proceed to close the writer.");
+ }
+ }
if (writer != null) {
writer.close();
}
super.serviceStop();
}
+
+ @VisibleForTesting
+ boolean writerFlusherRunning() {
+ return writerFlusherRunning;
+ }
+
+ /**
+ * Task that invokes the flush operation on the timeline writer.
+ */
+ private static class WriterFlushTask implements Runnable {
+ private final TimelineWriter writer;
+
+ public WriterFlushTask(TimelineWriter writer) {
+ this.writer = writer;
+ }
+
+ public void run() {
+ try {
+ writer.flush();
+ } catch (Throwable th) {
+ // we need to handle all exceptions or subsequent execution may be
+ // suppressed
+ LOG.error("exception during timeline writer flush!", th);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b2cff69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index 34a6b7c..b22b39f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -127,6 +127,11 @@ public class FileSystemTimelineWriterImpl extends AbstractService
mkdirs(outputRoot, ENTITIES_DIR);
}
+ @Override
+ public void flush() throws IOException {
+ // no op
+ }
+
private static String mkdirs(String... dirStrs) throws IOException {
StringBuilder path = new StringBuilder();
for (String dirStr : dirStrs) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b2cff69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index e48ca60..876ad6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -214,6 +214,12 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
return null;
}
+ @Override
+ public void flush() throws IOException {
+ // flush all buffered mutators
+ entityTable.flush();
+ }
+
/**
* close the hbase connections The close APIs perform flushing and release any
* resources held
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b2cff69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
index 5b4442c..381ff17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
@@ -187,6 +187,11 @@ public class PhoenixTimelineWriterImpl extends AbstractService
}
+ @Override
+ public void flush() throws IOException {
+ // currently no-op
+ }
+
// Utility functions
@Private
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b2cff69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
index 494e8ad..50136de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
@@ -70,4 +70,13 @@ public interface TimelineWriter extends Service {
*/
TimelineWriteResponse aggregate(TimelineEntity data,
TimelineAggregationTrack track) throws IOException;
+
+ /**
+ * Flushes the data to the backend storage. Whatever may be buffered will be
+ * written to the storage when the method returns. This may be a potentially
+ * time-consuming operation, and should be used judiciously.
+ *
+ * @throws IOException
+ */
+ void flush() throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b2cff69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
index 87343fd..0d69fbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
@@ -67,6 +67,11 @@ public class TestNMTimelineCollectorManager {
}
@Test
+ public void testStartingWriterFlusher() throws Exception {
+ assertTrue(collectorManager.writerFlusherRunning());
+ }
+
+ @Test
public void testStartWebApp() throws Exception {
assertNotNull(collectorManager.getRestServerBindAddress());
String address = collectorManager.getRestServerBindAddress();