You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by va...@apache.org on 2017/08/30 20:15:33 UTC
[08/50] [abbrv] hadoop git commit: YARN-4455. Support fetching
metrics by time range. Contributed by Varun Saxena.
YARN-4455. Support fetching metrics by time range. Contributed by Varun Saxena.
(cherry picked from commit 743e4731781a7d9a4a5f1f09adc510f193182158)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f66c16b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f66c16b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f66c16b
Branch: refs/heads/YARN-5355_branch2
Commit: 1f66c16ba6a7eb5bc522281c2ace05ea426a72cf
Parents: 45dcd81
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Thu Jul 20 12:16:06 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Thu Aug 31 01:10:04 2017 +0530
----------------------------------------------------------------------
...stTimelineReaderWebServicesHBaseStorage.java | 207 ++++++++++++++++-
.../storage/DataGeneratorForTest.java | 27 +--
.../storage/TestHBaseTimelineStorageApps.java | 209 ++++++++++++-----
.../TestHBaseTimelineStorageEntities.java | 166 ++++++++++----
.../storage/flow/TestHBaseStorageFlowRun.java | 18 +-
.../storage/common/ColumnHelper.java | 1 -
.../common/HBaseTimelineStorageUtils.java | 17 ++
.../storage/reader/ApplicationEntityReader.java | 14 ++
.../storage/reader/GenericEntityReader.java | 12 +-
.../reader/TimelineDataToRetrieve.java | 35 ++-
.../reader/TimelineReaderWebServices.java | 226 ++++++++++++++++---
.../reader/TimelineReaderWebServicesUtils.java | 6 +-
.../TestFileSystemTimelineReaderImpl.java | 15 +-
13 files changed, 777 insertions(+), 176 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f66c16b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index 6836cc1..b36eb9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -328,7 +328,7 @@ public class TestTimelineReaderWebServicesHBaseStorage
userEntity.setType("entitytype");
userEntity.setId("entityid-" + i);
userEntity.setIdPrefix(11 - i);
- userEntity.setCreatedTime(System.currentTimeMillis());
+ userEntity.setCreatedTime(ts);
userEntities.addEntity(userEntity);
}
@@ -344,7 +344,7 @@ public class TestTimelineReaderWebServicesHBaseStorage
flowVersion2, runid2, entity3.getId(), te3);
hbi.write(cluster, user, flow, flowVersion, runid,
"application_1111111111_1111", userEntities);
- writeApplicationEntities(hbi);
+ writeApplicationEntities(hbi, ts);
hbi.flush();
} finally {
if (hbi != null) {
@@ -353,26 +353,25 @@ public class TestTimelineReaderWebServicesHBaseStorage
}
}
- static void writeApplicationEntities(HBaseTimelineWriterImpl hbi)
- throws IOException {
- long currentTimeMillis = System.currentTimeMillis();
+ static void writeApplicationEntities(HBaseTimelineWriterImpl hbi,
+ long timestamp) throws IOException {
int count = 1;
for (long i = 1; i <= 3; i++) {
for (int j = 1; j <= 5; j++) {
TimelineEntities te = new TimelineEntities();
ApplicationId appId =
- BuilderUtils.newApplicationId(currentTimeMillis, count++);
+ BuilderUtils.newApplicationId(timestamp, count++);
ApplicationEntity appEntity = new ApplicationEntity();
appEntity.setId(appId.toString());
- appEntity.setCreatedTime(currentTimeMillis);
+ appEntity.setCreatedTime(timestamp);
TimelineEvent created = new TimelineEvent();
created.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- created.setTimestamp(currentTimeMillis);
+ created.setTimestamp(timestamp);
appEntity.addEvent(created);
TimelineEvent finished = new TimelineEvent();
finished.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- finished.setTimestamp(currentTimeMillis + i * j);
+ finished.setTimestamp(timestamp + i * j);
appEntity.addEvent(finished);
te.addEntity(appEntity);
@@ -1766,6 +1765,113 @@ public class TestTimelineReaderWebServicesHBaseStorage
}
}
+ private static void verifyMetricCount(TimelineEntity entity,
+ int expectedMetricsCnt, int expectedMeticsValCnt) {
+ int metricsValCnt = 0;
+ for (TimelineMetric m : entity.getMetrics()) {
+ metricsValCnt += m.getValues().size();
+ }
+ assertEquals(expectedMetricsCnt, entity.getMetrics().size());
+ assertEquals(expectedMeticsValCnt, metricsValCnt);
+ }
+
+ private static void verifyMetricsCount(Set<TimelineEntity> entities,
+ int expectedMetricsCnt, int expectedMeticsValCnt) {
+ int metricsCnt = 0;
+ int metricsValCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricsCnt += entity.getMetrics().size();
+ for (TimelineMetric m : entity.getMetrics()) {
+ metricsValCnt += m.getValues().size();
+ }
+ }
+ assertEquals(expectedMetricsCnt, metricsCnt);
+ assertEquals(expectedMeticsValCnt, metricsValCnt);
+ }
+
+ @Test
+ public void testGetEntitiesMetricsTimeRange() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?fields=ALL&metricslimit=100&metricstimestart=" +
+ (ts - 90000) + "&metricstimeend=" + (ts - 80000));
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ verifyMetricsCount(entities, 4, 4);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?fields=ALL&metricslimit=100&metricstimestart=" +
+ (ts - 100000) + "&metricstimeend=" + (ts - 80000));
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ verifyMetricsCount(entities, 5, 9);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?fields=ALL&metricslimit=100&metricstimestart=" +
+ (ts - 100000));
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ verifyMetricsCount(entities, 5, 9);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?fields=ALL&metricslimit=100&metricstimeend=" +
+ (ts - 90000));
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ verifyMetricsCount(entities, 5, 5);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?fields=ALL&metricstimestart=" +
+ (ts - 100000));
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ verifyMetricsCount(entities, 5, 5);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1/entity2?fields=ALL&metricstimestart=" +
+ (ts - 100000) + "&metricstimeend=" + (ts - 80000));
+ resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ verifyMetricCount(entity, 3, 3);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1/entity2?fields=ALL&metricslimit=5&metricstimestart=" +
+ (ts - 100000) + "&metricstimeend=" + (ts - 80000));
+ resp = getResponse(client, uri);
+ entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ verifyMetricCount(entity, 3, 5);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?fields=ALL&metricslimit=100&metricstimestart=" +
+ (ts - 80000) + "&metricstimeend=" + (ts - 90000));
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+ } finally {
+ client.destroy();
+ }
+ }
+
/**
* Tests if specific configs and metrics are retrieve for getEntity call.
*/
@@ -2366,4 +2472,87 @@ public class TestTimelineReaderWebServicesHBaseStorage
client.destroy();
}
}
+
+ @Test
+ public void testGetAppsMetricsRange() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" +
+ "1002345678919/apps?fields=ALL&metricslimit=100&metricstimestart=" +
+ (ts - 200000) + "&metricstimeend=" + (ts - 100000));
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ verifyMetricsCount(entities, 4, 4);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" +
+ "1002345678919/apps?fields=ALL&metricslimit=100");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ verifyMetricsCount(entities, 4, 10);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/" +
+ "apps?fields=ALL&metricslimit=100&metricstimestart=" +
+ (ts - 200000) + "&metricstimeend=" + (ts - 100000));
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(3, entities.size());
+ verifyMetricsCount(entities, 5, 5);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/" +
+ "apps?fields=ALL&metricslimit=100");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(3, entities.size());
+ verifyMetricsCount(entities, 5, 12);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" +
+ "1002345678919/apps?fields=ALL&metricslimit=100&metricstimestart=" +
+ (ts - 200000));
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ verifyMetricsCount(entities, 4, 10);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" +
+ "1002345678919/apps?fields=ALL&metricslimit=100&metricstimeend=" +
+ (ts - 100000));
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ verifyMetricsCount(entities, 4, 4);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/apps/application_1111111111_1111?userid=user1&fields=ALL" +
+ "&flowname=flow_name&flowrunid=1002345678919&metricslimit=100" +
+ "&metricstimestart=" +(ts - 200000) + "&metricstimeend=" +
+ (ts - 100000));
+ resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ verifyMetricCount(entity, 3, 3);
+
+ uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/" +
+ "apps?fields=ALL&metricslimit=100&metricstimestart=" +
+ (ts - 100000) + "&metricstimeend=" + (ts - 200000));
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+ } finally {
+ client.destroy();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f66c16b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
index 1a522fa..926d8bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
@@ -58,7 +58,8 @@ public final class DataGeneratorForTest {
TimelineSchemaCreator.createAllTables(conf, false);
}
- public static void loadApps(HBaseTestingUtility util) throws IOException {
+ public static void loadApps(HBaseTestingUtility util, long ts)
+ throws IOException {
TimelineEntities te = new TimelineEntities();
TimelineEntity entity = new TimelineEntity();
String id = "application_1111111111_2222";
@@ -92,7 +93,6 @@ public final class DataGeneratorForTest {
entity.addConfigs(conf);
// add metrics
Set<TimelineMetric> metrics = new HashSet<>();
- long ts = System.currentTimeMillis();
metrics.add(getMetric4(ts));
TimelineMetric m12 = new TimelineMetric();
@@ -137,7 +137,7 @@ public final class DataGeneratorForTest {
entity1.addConfigs(conf1);
// add metrics
- entity1.addMetrics(getMetrics4());
+ entity1.addMetrics(getMetrics4(ts));
TimelineEvent event11 = new TimelineEvent();
event11.setId("end_event");
event11.setTimestamp(ts);
@@ -175,18 +175,17 @@ public final class DataGeneratorForTest {
}
}
- private static Set<TimelineMetric> getMetrics4() {
+ private static Set<TimelineMetric> getMetrics4(long ts) {
Set<TimelineMetric> metrics1 = new HashSet<>();
TimelineMetric m2 = new TimelineMetric();
m2.setId("MAP1_SLOT_MILLIS");
- long ts1 = System.currentTimeMillis();
Map<Long, Number> metricValues1 = new HashMap<>();
- metricValues1.put(ts1 - 120000, 100000000);
- metricValues1.put(ts1 - 100000, 200000000);
- metricValues1.put(ts1 - 80000, 300000000);
- metricValues1.put(ts1 - 60000, 400000000);
- metricValues1.put(ts1 - 40000, 50000000000L);
- metricValues1.put(ts1 - 20000, 60000000000L);
+ metricValues1.put(ts - 120000, 100000000);
+ metricValues1.put(ts - 100000, 200000000);
+ metricValues1.put(ts - 80000, 300000000);
+ metricValues1.put(ts - 60000, 400000000);
+ metricValues1.put(ts - 40000, 50000000000L);
+ metricValues1.put(ts - 20000, 60000000000L);
m2.setType(Type.TIME_SERIES);
m2.setValues(metricValues1);
metrics1.add(m2);
@@ -307,7 +306,7 @@ public final class DataGeneratorForTest {
return metricValues;
}
- public static void loadEntities(HBaseTestingUtility util)
+ public static void loadEntities(HBaseTestingUtility util, long ts)
throws IOException {
TimelineEntities te = new TimelineEntities();
TimelineEntity entity = new TimelineEntity();
@@ -332,7 +331,6 @@ public final class DataGeneratorForTest {
Set<TimelineMetric> metrics = new HashSet<>();
TimelineMetric m1 = new TimelineMetric();
m1.setId("MAP_SLOT_MILLIS");
- long ts = System.currentTimeMillis();
m1.setType(Type.TIME_SERIES);
m1.setValues(getMetricValues1(ts));
metrics.add(m1);
@@ -383,9 +381,8 @@ public final class DataGeneratorForTest {
Set<TimelineMetric> metrics1 = new HashSet<>();
TimelineMetric m2 = new TimelineMetric();
m2.setId("MAP1_SLOT_MILLIS");
- long ts1 = System.currentTimeMillis();
m2.setType(Type.TIME_SERIES);
- m2.setValues(getMetricValues2(ts1));
+ m2.setValues(getMetricValues2(ts));
metrics1.add(m2);
entity1.addMetrics(metrics1);
te.addEntity(entity1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f66c16b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
index 321b72e..d6b0370 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
@@ -86,13 +86,14 @@ public class TestHBaseTimelineStorageApps {
private static HBaseTestingUtility util;
private HBaseTimelineReaderImpl reader;
+ private static final long CURRENT_TIME = System.currentTimeMillis();
@BeforeClass
public static void setupBeforeClass() throws Exception {
util = new HBaseTestingUtility();
util.startMiniCluster();
DataGeneratorForTest.createSchema(util.getConfiguration());
- DataGeneratorForTest.loadApps(util);
+ DataGeneratorForTest.loadApps(util, CURRENT_TIME);
}
@Before
@@ -235,13 +236,12 @@ public class TestHBaseTimelineStorageApps {
TimelineMetric m1 = new TimelineMetric();
m1.setId("MAP_SLOT_MILLIS");
Map<Long, Number> metricValues = new HashMap<Long, Number>();
- long ts = System.currentTimeMillis();
- metricValues.put(ts - 120000, 100000000);
- metricValues.put(ts - 100000, 200000000);
- metricValues.put(ts - 80000, 300000000);
- metricValues.put(ts - 60000, 400000000);
- metricValues.put(ts - 40000, 50000000000L);
- metricValues.put(ts - 20000, 60000000000L);
+ metricValues.put(CURRENT_TIME - 120000, 100000000);
+ metricValues.put(CURRENT_TIME - 100000, 200000000);
+ metricValues.put(CURRENT_TIME - 80000, 300000000);
+ metricValues.put(CURRENT_TIME - 60000, 400000000);
+ metricValues.put(CURRENT_TIME - 40000, 50000000000L);
+ metricValues.put(CURRENT_TIME - 20000, 60000000000L);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
@@ -258,7 +258,7 @@ public class TestHBaseTimelineStorageApps {
TimelineMetric aggMetric = new TimelineMetric();
aggMetric.setId("MEM_USAGE");
Map<Long, Number> aggMetricValues = new HashMap<Long, Number>();
- long aggTs = ts;
+ long aggTs = CURRENT_TIME;
aggMetricValues.put(aggTs - 120000, 102400000L);
aggMetric.setType(Type.SINGLE_VALUE);
aggMetric.setRealtimeAggregationOp(TimelineMetricOperation.SUM);
@@ -379,7 +379,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineReaderContext(cluster, user, flow, runid, appId,
entity.getType(), entity.getId()),
new TimelineDataToRetrieve(null, null,
- EnumSet.of(TimelineReader.Field.ALL), Integer.MAX_VALUE));
+ EnumSet.of(TimelineReader.Field.ALL), Integer.MAX_VALUE, null, null));
assertNotNull(e1);
// verify attributes
@@ -422,7 +422,7 @@ public class TestHBaseTimelineStorageApps {
e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow,
runid, appId, entity.getType(), entity.getId()),
new TimelineDataToRetrieve(null, null,
- EnumSet.of(TimelineReader.Field.ALL), 3));
+ EnumSet.of(TimelineReader.Field.ALL), 3, null, null));
assertNotNull(e1);
assertEquals(appId, e1.getId());
assertEquals(TimelineEntityType.YARN_APPLICATION.toString(),
@@ -443,7 +443,7 @@ public class TestHBaseTimelineStorageApps {
e1 = reader.getEntity(
new TimelineReaderContext(cluster, user, flow, runid, appId,
entity.getType(), entity.getId()), new TimelineDataToRetrieve(
- null, null, EnumSet.of(TimelineReader.Field.ALL), null));
+ null, null, EnumSet.of(TimelineReader.Field.ALL), null, null, null));
assertNotNull(e1);
assertEquals(appId, e1.getId());
assertEquals(TimelineEntityType.YARN_APPLICATION.toString(),
@@ -464,9 +464,9 @@ public class TestHBaseTimelineStorageApps {
metric.getId().equals("MEM_USAGE"));
assertEquals(1, metric.getValues().size());
if (metric.getId().equals("MAP_SLOT_MILLIS")) {
- assertTrue(metric.getValues().containsKey(ts - 20000));
- assertEquals(metricValues.get(ts - 20000),
- metric.getValues().get(ts - 20000));
+ assertTrue(metric.getValues().containsKey(CURRENT_TIME - 20000));
+ assertEquals(metricValues.get(CURRENT_TIME - 20000),
+ metric.getValues().get(CURRENT_TIME - 20000));
}
if (metric.getId().equals("MEM_USAGE")) {
assertTrue(metric.getValues().containsKey(aggTs - 120000));
@@ -552,11 +552,13 @@ public class TestHBaseTimelineStorageApps {
TimelineEntity e1 = reader.getEntity(
new TimelineReaderContext(cluster, user, flow, runid, appName,
entity.getType(), entity.getId()),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
TimelineEntity e2 = reader.getEntity(
new TimelineReaderContext(cluster, user, null, null, appName,
entity.getType(), entity.getId()),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertNotNull(e1);
assertNotNull(e2);
assertEquals(e1, e2);
@@ -650,7 +652,8 @@ public class TestHBaseTimelineStorageApps {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1111111111_2222",
TimelineEntityType.YARN_APPLICATION.toString(), null),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertNotNull(entity);
assertEquals(3, entity.getConfigs().size());
assertEquals(1, entity.getIsRelatedToEntities().size());
@@ -659,7 +662,8 @@ public class TestHBaseTimelineStorageApps {
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
null),
new TimelineEntityFilters.Builder().build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(3, entities.size());
int cfgCnt = 0;
int metricCnt = 0;
@@ -773,17 +777,17 @@ public class TestHBaseTimelineStorageApps {
1002345678919L, "application_1111111111_2222",
TimelineEntityType.YARN_APPLICATION.toString(), null),
new TimelineDataToRetrieve(
- null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null));
+ null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null, null, null));
assertNotNull(e1);
assertEquals(3, e1.getConfigs().size());
assertEquals(0, e1.getIsRelatedToEntities().size());
Set<TimelineEntity> es1 = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
- null),
- new TimelineEntityFilters.Builder().build(),
+ null), new TimelineEntityFilters.Builder().build(),
new TimelineDataToRetrieve(
- null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS), null));
+ null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS), null,
+ null, null));
assertEquals(3, es1.size());
int metricsCnt = 0;
int isRelatedToCnt = 0;
@@ -812,7 +816,8 @@ public class TestHBaseTimelineStorageApps {
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
null),
new TimelineEntityFilters.Builder().isRelatedTo(irt).build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(2, entities.size());
int isRelatedToCnt = 0;
for (TimelineEntity timelineEntity : entities) {
@@ -964,7 +969,8 @@ public class TestHBaseTimelineStorageApps {
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
null),
new TimelineEntityFilters.Builder().relatesTo(rt).build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(2, entities.size());
int relatesToCnt = 0;
for (TimelineEntity timelineEntity : entities) {
@@ -1202,7 +1208,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().configFilters(confFilterList)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(2, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1216,7 +1222,8 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().configFilters(confFilterList)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(2, entities.size());
cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1234,7 +1241,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().configFilters(confFilterList1)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(1, entities.size());
cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1254,7 +1261,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().configFilters(confFilterList2)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList confFilterList3 = new TimelineFilterList(
@@ -1267,7 +1274,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().configFilters(confFilterList3)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList confFilterList4 = new TimelineFilterList(
@@ -1280,7 +1287,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().configFilters(confFilterList4)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList confFilterList5 = new TimelineFilterList(
@@ -1293,7 +1300,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().configFilters(confFilterList5)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(3, entities.size());
}
@@ -1309,7 +1316,8 @@ public class TestHBaseTimelineStorageApps {
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
null),
new TimelineEntityFilters.Builder().eventFilters(ef).build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(1, entities.size());
int eventCnt = 0;
for (TimelineEntity timelineEntity : entities) {
@@ -1431,7 +1439,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1111111111_2222",
TimelineEntityType.YARN_APPLICATION.toString(), null),
- new TimelineDataToRetrieve(list, null, null, null));
+ new TimelineDataToRetrieve(list, null, null, null, null, null));
assertNotNull(e1);
assertEquals(1, e1.getConfigs().size());
Set<TimelineEntity> es1 = reader.getEntities(
@@ -1439,7 +1447,7 @@ public class TestHBaseTimelineStorageApps {
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
null) ,
new TimelineEntityFilters.Builder().build(),
- new TimelineDataToRetrieve(list, null, null, null));
+ new TimelineDataToRetrieve(list, null, null, null, null, null));
int cfgCnt = 0;
for (TimelineEntity entity : es1) {
cfgCnt += entity.getConfigs().size();
@@ -1465,7 +1473,7 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().configFilters(confFilterList)
.build(),
- new TimelineDataToRetrieve(list, null, null, null));
+ new TimelineDataToRetrieve(list, null, null, null, null, null));
assertEquals(1, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1498,7 +1506,8 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().configFilters(confFilterList1)
.build(),
- new TimelineDataToRetrieve(confsToRetrieve, null, null, null));
+ new TimelineDataToRetrieve(confsToRetrieve, null, null, null, null,
+ null));
assertEquals(2, entities.size());
cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1530,7 +1539,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(2, entities.size());
int metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1544,7 +1553,8 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().metricFilters(metricFilterList)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(2, entities.size());
metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1564,7 +1574,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList1)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(1, entities.size());
metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1584,7 +1594,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList2)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList metricFilterList3 = new TimelineFilterList(
@@ -1597,7 +1607,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList3)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList metricFilterList4 = new TimelineFilterList(
@@ -1610,7 +1620,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList4)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList metricFilterList5 = new TimelineFilterList(
@@ -1623,7 +1633,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList5)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(3, entities.size());
}
@@ -1636,7 +1646,7 @@ public class TestHBaseTimelineStorageApps {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1111111111_2222",
TimelineEntityType.YARN_APPLICATION.toString(), null),
- new TimelineDataToRetrieve(null, list, null, null));
+ new TimelineDataToRetrieve(null, list, null, null, null, null));
assertNotNull(e1);
assertEquals(1, e1.getMetrics().size());
Set<TimelineEntity> es1 = reader.getEntities(
@@ -1644,7 +1654,7 @@ public class TestHBaseTimelineStorageApps {
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
null),
new TimelineEntityFilters.Builder().build(),
- new TimelineDataToRetrieve(null, list, null, null));
+ new TimelineDataToRetrieve(null, list, null, null, null, null));
int metricCnt = 0;
for (TimelineEntity entity : es1) {
metricCnt += entity.getMetrics().size();
@@ -1670,7 +1680,7 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().metricFilters(metricFilterList)
.build(),
- new TimelineDataToRetrieve(null, list, null, null));
+ new TimelineDataToRetrieve(null, list, null, null, null, null));
int metricCnt = 0;
assertEquals(1, entities.size());
for (TimelineEntity entity : entities) {
@@ -1696,7 +1706,8 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().metricFilters(metricFilterList1)
.build(),
- new TimelineDataToRetrieve(null, metricsToRetrieve, null, null));
+ new TimelineDataToRetrieve(null, metricsToRetrieve, null, null, null,
+ null));
metricCnt = 0;
assertEquals(2, entities.size());
for (TimelineEntity entity : entities) {
@@ -1713,8 +1724,8 @@ public class TestHBaseTimelineStorageApps {
TimelineEntityType.YARN_APPLICATION.toString(), null),
new TimelineEntityFilters.Builder().metricFilters(metricFilterList1)
.build(),
- new TimelineDataToRetrieve(null,
- metricsToRetrieve, EnumSet.of(Field.METRICS), Integer.MAX_VALUE));
+ new TimelineDataToRetrieve(null, metricsToRetrieve,
+ EnumSet.of(Field.METRICS), Integer.MAX_VALUE, null, null));
metricCnt = 0;
int metricValCnt = 0;
assertEquals(2, entities.size());
@@ -1731,6 +1742,86 @@ public class TestHBaseTimelineStorageApps {
}
@Test
+ public void testReadAppsMetricTimeRange() throws Exception {
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null), new TimelineEntityFilters.Builder().build(),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
+ 100, null, null));
+ assertEquals(3, entities.size());
+ int metricTimeSeriesCnt = 0;
+ int metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric m : entity.getMetrics()) {
+ metricTimeSeriesCnt += m.getValues().size();
+ }
+ }
+ assertEquals(3, metricCnt);
+ assertEquals(13, metricTimeSeriesCnt);
+
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null), new TimelineEntityFilters.Builder().build(),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
+ 100, CURRENT_TIME - 40000, CURRENT_TIME));
+ assertEquals(3, entities.size());
+ metricCnt = 0;
+ metricTimeSeriesCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric m : entity.getMetrics()) {
+ for (Long ts : m.getValues().keySet()) {
+ assertTrue(ts >= CURRENT_TIME - 40000 && ts <= CURRENT_TIME);
+ }
+ metricTimeSeriesCnt += m.getValues().size();
+ }
+ }
+ assertEquals(3, metricCnt);
+ assertEquals(5, metricTimeSeriesCnt);
+
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null), new TimelineEntityFilters.Builder().build(),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
+ null, CURRENT_TIME - 40000, CURRENT_TIME));
+ assertEquals(3, entities.size());
+ metricCnt = 0;
+ metricTimeSeriesCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric m : entity.getMetrics()) {
+ for (Long ts : m.getValues().keySet()) {
+ assertTrue(ts >= CURRENT_TIME - 40000 && ts <= CURRENT_TIME);
+ }
+ metricTimeSeriesCnt += m.getValues().size();
+ }
+ }
+ assertEquals(3, metricCnt);
+ assertEquals(3, metricTimeSeriesCnt);
+
+ TimelineEntity entity = reader.getEntity(new TimelineReaderContext(
+ "cluster1", "user1", "some_flow_name", 1002345678919L,
+ "application_1111111111_2222",
+ TimelineEntityType.YARN_APPLICATION.toString(), null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), 100,
+ CURRENT_TIME - 40000, CURRENT_TIME));
+ assertNotNull(entity);
+ assertEquals(2, entity.getMetrics().size());
+ metricTimeSeriesCnt = 0;
+ for (TimelineMetric m : entity.getMetrics()) {
+ for (Long ts : m.getValues().keySet()) {
+ assertTrue(ts >= CURRENT_TIME - 40000 && ts <= CURRENT_TIME);
+ }
+ metricTimeSeriesCnt += m.getValues().size();
+ }
+ assertEquals(3, metricTimeSeriesCnt);
+ }
+
+ @Test
public void testReadAppsInfoFilters() throws Exception {
TimelineFilterList list1 = new TimelineFilterList();
list1.addFilter(new TimelineKeyValueFilter(
@@ -1749,7 +1840,8 @@ public class TestHBaseTimelineStorageApps {
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList).build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(2, entities.size());
int infoCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1766,7 +1858,8 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList1)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(1, entities.size());
infoCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1785,7 +1878,8 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList2)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(0, entities.size());
TimelineFilterList infoFilterList3 = new TimelineFilterList(
@@ -1797,7 +1891,8 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList3)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(0, entities.size());
TimelineFilterList infoFilterList4 = new TimelineFilterList(
@@ -1809,7 +1904,8 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList4)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(0, entities.size());
TimelineFilterList infoFilterList5 = new TimelineFilterList(
@@ -1821,7 +1917,8 @@ public class TestHBaseTimelineStorageApps {
null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList5)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(3, entities.size());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f66c16b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
index 9b47381..7ac5b36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
@@ -93,13 +93,14 @@ public class TestHBaseTimelineStorageEntities {
private static HBaseTestingUtility util;
private HBaseTimelineReaderImpl reader;
+ private static final long CURRENT_TIME = System.currentTimeMillis();
@BeforeClass
public static void setupBeforeClass() throws Exception {
util = new HBaseTestingUtility();
util.startMiniCluster();
DataGeneratorForTest.createSchema(util.getConfiguration());
- DataGeneratorForTest.loadEntities(util);
+ DataGeneratorForTest.loadEntities(util, CURRENT_TIME);
}
@Before
@@ -294,13 +295,13 @@ public class TestHBaseTimelineStorageEntities {
new TimelineReaderContext(cluster, user, flow, runid, appName,
entity.getType(), entity.getId()),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL),
- Integer.MAX_VALUE));
+ Integer.MAX_VALUE, null, null));
Set<TimelineEntity> es1 = reader.getEntities(
new TimelineReaderContext(cluster, user, flow, runid, appName,
entity.getType(), null),
new TimelineEntityFilters.Builder().build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL),
- Integer.MAX_VALUE));
+ Integer.MAX_VALUE, null, null));
assertNotNull(e1);
assertEquals(1, es1.size());
@@ -331,7 +332,8 @@ public class TestHBaseTimelineStorageEntities {
e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow,
runid, appName, entity.getType(), entity.getId()),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertNotNull(e1);
assertEquals(id, e1.getId());
assertEquals(type, e1.getType());
@@ -449,12 +451,14 @@ public class TestHBaseTimelineStorageEntities {
TimelineEntity e1 = reader.getEntity(
new TimelineReaderContext(cluster, user, flow, runid, appName,
entity.getType(), entity.getId()),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
Set<TimelineEntity> es1 = reader.getEntities(
new TimelineReaderContext(cluster, user, flow, runid, appName,
entity.getType(), null),
new TimelineEntityFilters.Builder().build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertNotNull(e1);
assertEquals(1, es1.size());
@@ -513,7 +517,8 @@ public class TestHBaseTimelineStorageEntities {
TimelineEntity e1 = reader.getEntity(
new TimelineReaderContext(cluster, user, flow, runid, appName,
entity.getType(), entity.getId()),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertNotNull(e1);
// check the events
NavigableSet<TimelineEvent> events = e1.getEvents();
@@ -542,7 +547,8 @@ public class TestHBaseTimelineStorageEntities {
TimelineEntity entity = reader.getEntity(
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", "hello"),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertNotNull(entity);
assertEquals(3, entity.getConfigs().size());
assertEquals(1, entity.getIsRelatedToEntities().size());
@@ -550,7 +556,8 @@ public class TestHBaseTimelineStorageEntities {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world",
null), new TimelineEntityFilters.Builder().build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(3, entities.size());
int cfgCnt = 0;
int metricCnt = 0;
@@ -677,7 +684,8 @@ public class TestHBaseTimelineStorageEntities {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().eventFilters(ef).build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(1, entities.size());
int eventCnt = 0;
for (TimelineEntity timelineEntity : entities) {
@@ -797,7 +805,8 @@ public class TestHBaseTimelineStorageEntities {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().isRelatedTo(irt).build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(2, entities.size());
int isRelatedToCnt = 0;
for (TimelineEntity timelineEntity : entities) {
@@ -939,7 +948,8 @@ public class TestHBaseTimelineStorageEntities {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().relatesTo(rt).build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(2, entities.size());
int relatesToCnt = 0;
for (TimelineEntity timelineEntity : entities) {
@@ -1134,7 +1144,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", "hello"),
new TimelineDataToRetrieve(
- null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null));
+ null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null, null, null));
assertNotNull(e1);
assertEquals(3, e1.getConfigs().size());
assertEquals(0, e1.getIsRelatedToEntities().size());
@@ -1142,8 +1152,8 @@ public class TestHBaseTimelineStorageEntities {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().build(),
- new TimelineDataToRetrieve(
- null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.IS_RELATED_TO,
+ Field.METRICS), null, null, null));
assertEquals(3, es1.size());
int metricsCnt = 0;
int isRelatedToCnt = 0;
@@ -1166,14 +1176,14 @@ public class TestHBaseTimelineStorageEntities {
TimelineEntity e1 = reader.getEntity(
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", "hello"),
- new TimelineDataToRetrieve(list, null, null, null));
+ new TimelineDataToRetrieve(list, null, null, null, null, null));
assertNotNull(e1);
assertEquals(1, e1.getConfigs().size());
Set<TimelineEntity> es1 = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().build(),
- new TimelineDataToRetrieve(list, null, null, null));
+ new TimelineDataToRetrieve(list, null, null, null, null, null));
int cfgCnt = 0;
for (TimelineEntity entity : es1) {
cfgCnt += entity.getConfigs().size();
@@ -1205,7 +1215,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().configFilters(confFilterList)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(2, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1218,7 +1228,8 @@ public class TestHBaseTimelineStorageEntities {
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().configFilters(confFilterList)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(2, entities.size());
cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1235,7 +1246,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().configFilters(confFilterList1)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(1, entities.size());
cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1254,7 +1265,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().configFilters(confFilterList2)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList confFilterList3 = new TimelineFilterList(
@@ -1266,7 +1277,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().configFilters(confFilterList3)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList confFilterList4 = new TimelineFilterList(
@@ -1278,7 +1289,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().configFilters(confFilterList4)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList confFilterList5 = new TimelineFilterList(
@@ -1290,7 +1301,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().configFilters(confFilterList5)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
- null));
+ null, null, null));
assertEquals(3, entities.size());
}
@@ -1307,7 +1318,7 @@ public class TestHBaseTimelineStorageEntities {
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().configFilters(confFilterList)
.build(),
- new TimelineDataToRetrieve(list, null, null, null));
+ new TimelineDataToRetrieve(list, null, null, null, null, null));
assertEquals(1, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1338,7 +1349,8 @@ public class TestHBaseTimelineStorageEntities {
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().configFilters(confFilterList1)
.build(),
- new TimelineDataToRetrieve(confsToRetrieve, null, null, null));
+ new TimelineDataToRetrieve(confsToRetrieve, null, null, null, null,
+ null));
assertEquals(2, entities.size());
cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1359,14 +1371,14 @@ public class TestHBaseTimelineStorageEntities {
TimelineEntity e1 = reader.getEntity(
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", "hello"),
- new TimelineDataToRetrieve(null, list, null, null));
+ new TimelineDataToRetrieve(null, list, null, null, null, null));
assertNotNull(e1);
assertEquals(1, e1.getMetrics().size());
Set<TimelineEntity> es1 = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().build(),
- new TimelineDataToRetrieve(null, list, null, null));
+ new TimelineDataToRetrieve(null, list, null, null, null, null));
int metricCnt = 0;
for (TimelineEntity entity : es1) {
metricCnt += entity.getMetrics().size();
@@ -1379,6 +1391,63 @@ public class TestHBaseTimelineStorageEntities {
}
@Test
+ public void testReadEntitiesMetricTimeRange() throws Exception {
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters.Builder().build(),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
+ 100, null, null));
+ assertEquals(3, entities.size());
+ int metricTimeSeriesCnt = 0;
+ int metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric m : entity.getMetrics()) {
+ metricTimeSeriesCnt += m.getValues().size();
+ }
+ }
+ assertEquals(3, metricCnt);
+ assertEquals(13, metricTimeSeriesCnt);
+
+ entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1",
+ "some_flow_name", 1002345678919L, "application_1231111111_1111",
+ "world", null), new TimelineEntityFilters.Builder().build(),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
+ 100, CURRENT_TIME - 40000, CURRENT_TIME));
+ assertEquals(3, entities.size());
+ metricCnt = 0;
+ metricTimeSeriesCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric m : entity.getMetrics()) {
+ for (Long ts : m.getValues().keySet()) {
+ assertTrue(ts >= CURRENT_TIME - 40000 && ts <= CURRENT_TIME);
+ }
+ metricTimeSeriesCnt += m.getValues().size();
+ }
+ }
+ assertEquals(3, metricCnt);
+ assertEquals(5, metricTimeSeriesCnt);
+
+ TimelineEntity entity = reader.getEntity(new TimelineReaderContext(
+ "cluster1", "user1", "some_flow_name", 1002345678919L,
+ "application_1231111111_1111", "world", "hello"),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), 100,
+ CURRENT_TIME - 40000, CURRENT_TIME));
+ assertNotNull(entity);
+ assertEquals(2, entity.getMetrics().size());
+ metricTimeSeriesCnt = 0;
+ for (TimelineMetric m : entity.getMetrics()) {
+ for (Long ts : m.getValues().keySet()) {
+ assertTrue(ts >= CURRENT_TIME - 40000 && ts <= CURRENT_TIME);
+ }
+ metricTimeSeriesCnt += m.getValues().size();
+ }
+ assertEquals(3, metricTimeSeriesCnt);
+ }
+
+ @Test
public void testReadEntitiesMetricFilters() throws Exception {
TimelineFilterList list1 = new TimelineFilterList();
list1.addFilter(new TimelineCompareFilter(
@@ -1396,7 +1465,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(2, entities.size());
int metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1409,7 +1478,8 @@ public class TestHBaseTimelineStorageEntities {
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().metricFilters(metricFilterList)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
+ null, null));
assertEquals(2, entities.size());
metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1428,7 +1498,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList1)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(1, entities.size());
metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1447,7 +1517,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList2)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList metricFilterList3 = new TimelineFilterList(
@@ -1459,7 +1529,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList3)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList metricFilterList4 = new TimelineFilterList(
@@ -1471,7 +1541,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList4)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(0, entities.size());
TimelineFilterList metricFilterList5 = new TimelineFilterList(
@@ -1483,7 +1553,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList5)
.build(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
- null));
+ null, null, null));
assertEquals(3, entities.size());
}
@@ -1500,7 +1570,7 @@ public class TestHBaseTimelineStorageEntities {
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().metricFilters(metricFilterList)
.build(),
- new TimelineDataToRetrieve(null, list, null, null));
+ new TimelineDataToRetrieve(null, list, null, null, null, null));
assertEquals(1, entities.size());
int metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1530,7 +1600,7 @@ public class TestHBaseTimelineStorageEntities {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList1)
.build(),
new TimelineDataToRetrieve(
- null, metricsToRetrieve, EnumSet.of(Field.METRICS), null));
+ null, metricsToRetrieve, EnumSet.of(Field.METRICS), null, null, null));
assertEquals(2, entities.size());
metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1549,8 +1619,8 @@ public class TestHBaseTimelineStorageEntities {
"world", null),
new TimelineEntityFilters.Builder().metricFilters(metricFilterList1)
.build(),
- new TimelineDataToRetrieve(null,
- metricsToRetrieve, EnumSet.of(Field.METRICS), Integer.MAX_VALUE));
+ new TimelineDataToRetrieve(null, metricsToRetrieve,
+ EnumSet.of(Field.METRICS), Integer.MAX_VALUE, null, null));
assertEquals(2, entities.size());
metricCnt = 0;
int metricValCnt = 0;
@@ -1584,7 +1654,8 @@ public class TestHBaseTimelineStorageEntities {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList).build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(2, entities.size());
int infoCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1600,7 +1671,8 @@ public class TestHBaseTimelineStorageEntities {
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList1)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(1, entities.size());
infoCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1618,7 +1690,8 @@ public class TestHBaseTimelineStorageEntities {
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList2)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(0, entities.size());
TimelineFilterList infoFilterList3 = new TimelineFilterList(
@@ -1629,7 +1702,8 @@ public class TestHBaseTimelineStorageEntities {
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList3)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(0, entities.size());
TimelineFilterList infoFilterList4 = new TimelineFilterList(
@@ -1640,7 +1714,8 @@ public class TestHBaseTimelineStorageEntities {
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList4)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(0, entities.size());
TimelineFilterList infoFilterList5 = new TimelineFilterList(
@@ -1651,7 +1726,8 @@ public class TestHBaseTimelineStorageEntities {
1002345678919L, "application_1231111111_1111", "world", null),
new TimelineEntityFilters.Builder().infoFilters(infoFilterList5)
.build(),
- new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null,
+ null, null));
assertEquals(3, entities.size());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f66c16b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index e1309e7..acfdc4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -584,7 +584,8 @@ public class TestHBaseStorageFlowRun {
TimelineEntity entity = hbr.getEntity(
new TimelineReaderContext(cluster, user, flow, 1002345678919L, null,
TimelineEntityType.YARN_FLOW_RUN.toString(), null),
- new TimelineDataToRetrieve(null, metricsToRetrieve, null, null));
+ new TimelineDataToRetrieve(null, metricsToRetrieve, null, null, null,
+ null));
assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
Set<TimelineMetric> metrics = entity.getMetrics();
assertEquals(1, metrics.size());
@@ -609,7 +610,8 @@ public class TestHBaseStorageFlowRun {
new TimelineReaderContext(cluster, user, flow, null, null,
TimelineEntityType.YARN_FLOW_RUN.toString(), null),
new TimelineEntityFilters.Builder().build(),
- new TimelineDataToRetrieve(null, metricsToRetrieve, null, null));
+ new TimelineDataToRetrieve(null, metricsToRetrieve, null, null, null,
+ null));
assertEquals(2, entities.size());
int metricCnt = 0;
for (TimelineEntity timelineEntity : entities) {
@@ -681,7 +683,7 @@ public class TestHBaseStorageFlowRun {
TimelineEntityType.YARN_FLOW_RUN.toString(), null),
new TimelineEntityFilters.Builder().build(),
new TimelineDataToRetrieve(null, null,
- EnumSet.of(Field.METRICS), null));
+ EnumSet.of(Field.METRICS), null, null, null));
assertEquals(1, entities.size());
for (TimelineEntity timelineEntity : entities) {
Set<TimelineMetric> timelineMetrics = timelineEntity.getMetrics();
@@ -948,7 +950,7 @@ public class TestHBaseStorageFlowRun {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList)
.build(),
new TimelineDataToRetrieve(null, null,
- EnumSet.of(Field.METRICS), null));
+ EnumSet.of(Field.METRICS), null, null, null));
assertEquals(2, entities.size());
int metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -966,7 +968,7 @@ public class TestHBaseStorageFlowRun {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList1)
.build(),
new TimelineDataToRetrieve(null, null,
- EnumSet.of(Field.METRICS), null));
+ EnumSet.of(Field.METRICS), null, null, null));
assertEquals(1, entities.size());
metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -983,7 +985,7 @@ public class TestHBaseStorageFlowRun {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList2)
.build(),
new TimelineDataToRetrieve(null, null,
- EnumSet.of(Field.METRICS), null));
+ EnumSet.of(Field.METRICS), null, null, null));
assertEquals(0, entities.size());
TimelineFilterList metricFilterList3 = new TimelineFilterList(
@@ -994,7 +996,7 @@ public class TestHBaseStorageFlowRun {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList3)
.build(),
new TimelineDataToRetrieve(null, null,
- EnumSet.of(Field.METRICS), null));
+ EnumSet.of(Field.METRICS), null, null, null));
assertEquals(0, entities.size());
TimelineFilterList list3 = new TimelineFilterList();
@@ -1016,7 +1018,7 @@ public class TestHBaseStorageFlowRun {
new TimelineEntityFilters.Builder().metricFilters(metricFilterList4)
.build(),
new TimelineDataToRetrieve(null, metricsToRetrieve,
- EnumSet.of(Field.ALL), null));
+ EnumSet.of(Field.ALL), null, null, null));
assertEquals(2, entities.size());
metricCnt = 0;
for (TimelineEntity entity : entities) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f66c16b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
index b9815eb..496b0ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -190,7 +190,6 @@ public class ColumnHelper<T> {
NavigableMap<byte[], NavigableMap<Long, byte[]>> columnCellMap =
resultMap.get(columnFamilyBytes);
-
// could be that there is no such column family.
if (columnCellMap != null) {
for (Entry<byte[], NavigableMap<Long, byte[]>> entry : columnCellMap
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f66c16b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
index 865a70d..d3ef847 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.client.Query;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
@@ -308,4 +309,20 @@ public final class HBaseTimelineStorageUtils {
return (obj instanceof Short) || (obj instanceof Integer) ||
(obj instanceof Long);
}
+
+ public static void setMetricsTimeRange(Query query, byte[] metricsCf,
+ long tsBegin, long tsEnd) {
+ if (tsBegin != 0 || tsEnd != Long.MAX_VALUE) {
+ long supplementedTsBegin = tsBegin == 0 ? 0 :
+ TimestampGenerator.getSupplementedTimestamp(tsBegin, null);
+ long supplementedTsEnd =
+ (tsEnd == Long.MAX_VALUE) ? Long.MAX_VALUE :
+ TimestampGenerator.getSupplementedTimestamp(tsEnd + 1, null);
+ // Handle overflow by resetting time begin to 0 and time end to
+ // Long#MAX_VALUE, if required.
+ query.setColumnFamilyTimeRange(metricsCf,
+ ((supplementedTsBegin < 0) ? 0 : supplementedTsBegin),
+ ((supplementedTsEnd < 0) ? Long.MAX_VALUE : supplementedTsEnd));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f66c16b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
index b4bb005..cda4510 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Query;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -315,6 +316,8 @@ class ApplicationEntityReader extends GenericEntityReader {
context.getFlowName(), context.getFlowRunId(), context.getAppId());
byte[] rowKey = applicationRowKey.getRowKey();
Get get = new Get(rowKey);
+ // Set time range for metric values.
+ setMetricsTimeRange(get);
get.setMaxVersions(getDataToRetrieve().getMetricsLimit());
if (filterList != null && !filterList.getFilters().isEmpty()) {
get.setFilter(filterList);
@@ -357,6 +360,14 @@ class ApplicationEntityReader extends GenericEntityReader {
}
}
+ private void setMetricsTimeRange(Query query) {
+ // Set time range for metric values.
+ HBaseTimelineStorageUtils.
+ setMetricsTimeRange(query, ApplicationColumnFamily.METRICS.getBytes(),
+ getDataToRetrieve().getMetricsTimeBegin(),
+ getDataToRetrieve().getMetricsTimeEnd());
+ }
+
@Override
protected ResultScanner getResults(Configuration hbaseConf,
Connection conn, FilterList filterList) throws IOException {
@@ -405,6 +416,9 @@ class ApplicationEntityReader extends GenericEntityReader {
newList.addFilter(filterList);
}
scan.setFilter(newList);
+
+ // Set time range for metric values.
+ setMetricsTimeRange(scan);
scan.setMaxVersions(getDataToRetrieve().getMetricsLimit());
return getTable().getResultScanner(hbaseConf, conn, scan);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f66c16b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
index 39013d9..6b740e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Query;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -434,8 +435,8 @@ class GenericEntityReader extends TimelineEntityReader {
context.getUserId(), context.getFlowName(), context.getFlowRunId(),
context.getAppId(), context.getEntityType(),
context.getEntityIdPrefix(), context.getEntityId()).getRowKey();
-
Get get = new Get(rowKey);
+ setMetricsTimeRange(get);
get.setMaxVersions(getDataToRetrieve().getMetricsLimit());
if (filterList != null && !filterList.getFilters().isEmpty()) {
get.setFilter(filterList);
@@ -468,6 +469,14 @@ class GenericEntityReader extends TimelineEntityReader {
return result;
}
+ private void setMetricsTimeRange(Query query) {
+ // Set time range for metric values.
+ HBaseTimelineStorageUtils.
+ setMetricsTimeRange(query, EntityColumnFamily.METRICS.getBytes(),
+ getDataToRetrieve().getMetricsTimeBegin(),
+ getDataToRetrieve().getMetricsTimeEnd());
+ }
+
@Override
protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
FilterList filterList) throws IOException {
@@ -513,6 +522,7 @@ class GenericEntityReader extends TimelineEntityReader {
// mode.
filterList.addFilter(new PageFilter(getFilters().getLimit()));
}
+ setMetricsTimeRange(scan);
scan.setMaxVersions(getDataToRetrieve().getMetricsLimit());
if (filterList != null && !filterList.getFilters().isEmpty()) {
scan.setFilter(filterList);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f66c16b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
index 325050a..8d09c00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
@@ -57,6 +57,11 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Fiel
* metricsToRetrieve is specified, this limit defines an upper limit to the
* number of metrics to return. This parameter is ignored if METRICS are not to
* be fetched.</li>
+ * <li><b>metricsTimeStart</b> - Metric values before this timestamp would not
+ * be retrieved. If null or {@literal <0}, defaults to 0.</li>
+ * <li><b>metricsTimeEnd</b> - Metric values after this timestamp would not
+ * be retrieved. If null or {@literal <0}, defaults to {@link Long#MAX_VALUE}.
+ * </li>
* </ul>
*/
@Private
@@ -66,6 +71,10 @@ public class TimelineDataToRetrieve {
private TimelineFilterList metricsToRetrieve;
private EnumSet<Field> fieldsToRetrieve;
private Integer metricsLimit;
+ private Long metricsTimeBegin;
+ private Long metricsTimeEnd;
+ private static final long DEFAULT_METRICS_BEGIN_TIME = 0L;
+ private static final long DEFAULT_METRICS_END_TIME = Long.MAX_VALUE;
/**
* Default limit of number of metrics to return.
@@ -73,12 +82,12 @@ public class TimelineDataToRetrieve {
public static final Integer DEFAULT_METRICS_LIMIT = 1;
public TimelineDataToRetrieve() {
- this(null, null, null, null);
+ this(null, null, null, null, null, null);
}
public TimelineDataToRetrieve(TimelineFilterList confs,
TimelineFilterList metrics, EnumSet<Field> fields,
- Integer limitForMetrics) {
+ Integer limitForMetrics, Long metricTimeBegin, Long metricTimeEnd) {
this.confsToRetrieve = confs;
this.metricsToRetrieve = metrics;
this.fieldsToRetrieve = fields;
@@ -91,6 +100,20 @@ public class TimelineDataToRetrieve {
if (this.fieldsToRetrieve == null) {
this.fieldsToRetrieve = EnumSet.noneOf(Field.class);
}
+ if (metricTimeBegin == null || metricTimeBegin < 0) {
+ this.metricsTimeBegin = DEFAULT_METRICS_BEGIN_TIME;
+ } else {
+ this.metricsTimeBegin = metricTimeBegin;
+ }
+ if (metricTimeEnd == null || metricTimeEnd < 0) {
+ this.metricsTimeEnd = DEFAULT_METRICS_END_TIME;
+ } else {
+ this.metricsTimeEnd = metricTimeEnd;
+ }
+ if (this.metricsTimeBegin > this.metricsTimeEnd) {
+ throw new IllegalArgumentException("metricstimebegin should not be " +
+ "greater than metricstimeend");
+ }
}
public TimelineFilterList getConfsToRetrieve() {
@@ -137,6 +160,14 @@ public class TimelineDataToRetrieve {
return metricsLimit;
}
+ public Long getMetricsTimeBegin() {
+ return this.metricsTimeBegin;
+ }
+
+ public Long getMetricsTimeEnd() {
+ return metricsTimeEnd;
+ }
+
public void setMetricsLimit(Integer limit) {
if (limit == null || limit < 1) {
this.metricsLimit = DEFAULT_METRICS_LIMIT;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org