You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/09/16 17:07:41 UTC
incubator-metron git commit: METRON-413 Allow Start/End Time Range
Search in Profiler Client API (nickwallen) closes apache/incubator-metron#249
Repository: incubator-metron
Updated Branches:
refs/heads/master 3a2ecc404 -> 32d5ff64a
METRON-413 Allow Start/End Time Range Search in Profiler Client API (nickwallen) closes apache/incubator-metron#249
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/32d5ff64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/32d5ff64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/32d5ff64
Branch: refs/heads/master
Commit: 32d5ff64af0405ac99a718896ac15511046c2575
Parents: 3a2ecc4
Author: nickwallen <ni...@nickallen.org>
Authored: Fri Sep 16 12:42:59 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Fri Sep 16 12:42:59 2016 -0400
----------------------------------------------------------------------
.../profiler/client/HBaseProfilerClient.java | 27 ++++-
.../metron/profiler/client/ProfilerClient.java | 27 +++--
.../profiler/client/stellar/GetProfile.java | 2 +-
.../client/HBaseProfilerClientTest.java | 105 +++++++++++++++----
.../metron/profiler/hbase/RowKeyBuilder.java | 16 +--
.../profiler/hbase/SaltyRowKeyBuilder.java | 16 +--
.../profiler/hbase/SaltyRowKeyBuilderTest.java | 5 +-
7 files changed, 149 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
index cef2ea4..1d11f42 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
@@ -64,21 +64,40 @@ public class HBaseProfilerClient implements ProfilerClient {
/**
* Fetches all of the data values associated with a Profile.
*
+ * @param clazz The type of values stored by the profile.
* @param profile The name of the profile.
* @param entity The name of the entity.
+ * @param groups The groups used to sort the profile data.
* @param durationAgo How far in the past to fetch values from.
* @param unit The time unit of 'durationAgo'.
- * @param groups The groups
* @param <T> The type of values stored by the Profile.
- * @return A list of profile values.
+ * @return A list of values.
*/
@Override
- public <T> List<T> fetch(String profile, String entity, long durationAgo, TimeUnit unit, Class<T> clazz, List<Object> groups) {
+ public <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long durationAgo, TimeUnit unit) {
+ long end = System.currentTimeMillis();
+ long start = end - unit.toMillis(durationAgo);
+ return fetch(clazz, profile, entity, groups, start, end);
+ }
+
+ /**
+ * Fetch the values stored in a profile based on a start and end timestamp.
+ *
+ * @param clazz The type of values stored by the profile.
+ * @param profile The name of the profile.
+ * @param entity The name of the entity.
+ * @param groups The groups used to sort the profile data.
+ * @param start The start time in epoch milliseconds.
+ * @param end The end time in epoch milliseconds.
+ * @param <T> The type of values stored by the profile.
+ * @return A list of values.
+ */
+ public <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long start, long end) {
byte[] columnFamily = Bytes.toBytes(columnBuilder.getColumnFamily());
byte[] columnQualifier = columnBuilder.getColumnQualifier("value");
// find all the row keys that satisfy this fetch
- List<byte[]> keysToFetch = rowKeyBuilder.rowKeys(profile, entity, groups, durationAgo, unit);
+ List<byte[]> keysToFetch = rowKeyBuilder.rowKeys(profile, entity, groups, start, end);
// create a Get for each of the row keys
List<Get> gets = keysToFetch
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
index 9cae0e9..c6a5379 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
@@ -31,15 +31,28 @@ public interface ProfilerClient {
/**
* Fetch the measurement values associated with a profile.
*
- * @param profile The name of the profile.
- * @param entity The name of the entity.
+ * @param clazz The type of values stored by the profile.
+ * @param profile The name of the profile.
+ * @param entity The name of the entity.
+ * @param groups The groups used to sort the profile data.
* @param durationAgo How far in the past to fetch values from.
- * @param unit The time unit of 'durationAgo'.
- * @param clazz The type of values stored by the profile.
- * @param groups The groups used to sort the profile data.
- * @param <T> The type of values stored by the Profile.
+ * @param unit The time unit of 'durationAgo'.
+ * @param <T> The type of values stored by the Profile.
* @return A list of values.
*/
- <T> List<T> fetch(String profile, String entity, long durationAgo, TimeUnit unit, Class<T> clazz, List<Object> groups);
+ <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long durationAgo, TimeUnit unit);
+ /**
+ * Fetch the values stored in a profile based on a start and end timestamp.
+ *
+ * @param clazz The type of values stored by the profile.
+ * @param profile The name of the profile.
+ * @param entity The name of the entity.
+ * @param groups The groups used to sort the profile data.
+ * @param start The start time in epoch milliseconds.
+ * @param end The end time in epoch milliseconds.
+ * @param <T> The type of values stored by the profile.
+ * @return A list of values.
+ */
+ <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long start, long end);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
index d96419f..20546a2 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
@@ -142,7 +142,7 @@ public class GetProfile implements StellarFunction {
TimeUnit units = TimeUnit.valueOf(unitsName);
List<Object> groups = getGroupsArg(4, args);
- return client.fetch(profile, entity, durationAgo, units, Object.class, groups);
+ return client.fetch(Object.class, profile, entity, groups, durationAgo, units);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
index 0da3f31..0076396 100644
--- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
@@ -57,6 +57,9 @@ public class HBaseProfilerClientTest {
private static final String tableName = "profiler";
private static final String columnFamily = "P";
+ private static final long periodDuration = 15;
+ private static final TimeUnit periodUnits = TimeUnit.MINUTES;
+ private static final int periodsPerHour = 4;
private HBaseProfilerClient client;
private HTableInterface table;
@@ -103,24 +106,89 @@ public class HBaseProfilerClientTest {
* The client should be able to distinguish between groups and only fetch those in the correct group.
*/
@Test
- public void testFetchOneGroup() throws Exception {
+ public void testFetchWithDurationAgoAndOneGroup() throws Exception {
+ final int expectedValue = 2302;
+ final int hours = 2;
+ final int count = hours * periodsPerHour;
+ final long startTime = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(hours);
+
+ // setup - write two groups of measurements - 'weekends' and 'weekdays'
+ ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", startTime, periodDuration, periodUnits);
+ profileWriter.write(m, count, Arrays.asList("weekdays"), val -> expectedValue);
+ profileWriter.write(m, count, Arrays.asList("weekends"), val -> 0);
+
+ // execute
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", Arrays.asList("weekdays"), hours, TimeUnit.HOURS);
+
+ // validate
+ assertEquals(count, results.size());
+ results.forEach(actual -> assertEquals(expectedValue, (int) actual));
+ }
+
+ /**
+ * Attempt to fetch a group that does not exist.
+ */
+ @Test
+ public void testFetchWithDurationAgoAndNoGroup() {
// setup
- final long periodDuration = 15;
- final TimeUnit periodUnits = TimeUnit.MINUTES;
- final int periodsPerHour = 4;
final int expectedValue = 2302;
final int hours = 2;
- final int count = hours * periodsPerHour;
final long startTime = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(hours);
// create two groups of measurements - one on weekdays and one on weekends
ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", startTime, periodDuration, periodUnits);
+ profileWriter.write(m, hours * periodsPerHour, Arrays.asList("weekdays"), val -> expectedValue);
+ profileWriter.write(m, hours * periodsPerHour, Arrays.asList("weekends"), val -> 0);
+
+ // execute
+ List<Object> doesNotExist = Arrays.asList("does-not-exist");
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", doesNotExist, hours, TimeUnit.HOURS);
+
+ // validate
+ assertEquals(0, results.size());
+ }
+
+ /**
+ * Profile data only within 'milliseconds ago' should be fetched. Data outside of that time horizon should
+ * not be fetched.
+ */
+ @Test
+ public void testFetchWithDurationAgoAndOutsideTimeWindow() throws Exception {
+ final int hours = 2;
+ final List<Object> group = Arrays.asList("weekends");
+ final long startTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1);
+
+ // setup - write some values to read later
+ ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", startTime, periodDuration, periodUnits);
+ profileWriter.write(m, hours * periodsPerHour, group, val -> 1000);
+
+ // execute
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", group, 2, TimeUnit.MILLISECONDS);
+
+ // validate - there should NOT be any results from just 2 milliseconds ago
+ assertEquals(0, results.size());
+ }
+
+ /**
+ * The client should be able to distinguish between groups and only fetch those in the correct group.
+ */
+ @Test
+ public void testFetchWithStartEndAndOneGroup() throws Exception {
+ final int periodsPerHour = 4;
+ final int expectedValue = 2302;
+ final int hours = 2;
+ final int count = hours * periodsPerHour;
+ final long endTime = System.currentTimeMillis();
+ final long startTime = endTime - TimeUnit.HOURS.toMillis(hours);
+
+ // setup - write two groups of measurements - 'weekends' and 'weekdays'
+ ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", startTime, periodDuration, periodUnits);
profileWriter.write(m, count, Arrays.asList("weekdays"), val -> expectedValue);
profileWriter.write(m, count, Arrays.asList("weekends"), val -> 0);
// execute
- List<Integer> results = client.fetch("profile1", "entity1", hours, TimeUnit.HOURS, Integer.class, Arrays.asList("weekdays"));
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", Arrays.asList("weekdays"), startTime, endTime);
// validate
assertEquals(count, results.size());
@@ -131,16 +199,15 @@ public class HBaseProfilerClientTest {
* Attempt to fetch a group that does not exist.
*/
@Test
- public void testFetchNoGroup() {
+ public void testFetchWithStartEndAndNoGroup() {
// setup
- final long periodDuration = 15;
- final TimeUnit periodUnits = TimeUnit.MINUTES;
final int periodsPerHour = 4;
final int expectedValue = 2302;
final int hours = 2;
final int count = hours * periodsPerHour;
- final long startTime = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(hours);
+ final long endTime = System.currentTimeMillis();
+ final long startTime = endTime - TimeUnit.HOURS.toMillis(hours);
// create two groups of measurements - one on weekdays and one on weekends
ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", startTime, periodDuration, periodUnits);
@@ -149,7 +216,7 @@ public class HBaseProfilerClientTest {
// execute
List<Object> doesNotExist = Arrays.asList("does-not-exist");
- List<Integer> results = client.fetch("profile1", "entity1", hours, TimeUnit.HOURS, Integer.class, doesNotExist);
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", doesNotExist, startTime, endTime);
// validate
assertEquals(0, results.size());
@@ -160,23 +227,23 @@ public class HBaseProfilerClientTest {
* not be fetched.
*/
@Test
- public void testFetchOutsideTimeWindow() throws Exception {
- final long periodDuration = 15;
- final TimeUnit periodUnits = TimeUnit.MINUTES;
- final int periodsPerHour = 4;
+ public void testFetchWithStartEndAndOutsideTimeWindow() throws Exception {
+
final int hours = 2;
int numberToWrite = hours * periodsPerHour;
final List<Object> group = Arrays.asList("weekends");
- final long startTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1);
+ final long measurementTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1);
// setup - write some values to read later
- ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", startTime, periodDuration, periodUnits);
+ ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", measurementTime, periodDuration, periodUnits);
profileWriter.write(m, numberToWrite, group, val -> 1000);
// execute
- List<Integer> results = client.fetch("profile1", "entity1", 2, TimeUnit.MILLISECONDS, Integer.class, group);
+ final long endFetchAt = System.currentTimeMillis();
+ final long startFetchAt = endFetchAt - TimeUnit.MILLISECONDS.toMillis(30);
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", group, startFetchAt, endFetchAt);
// validate - there should NOT be any results from just 2 milliseconds ago
assertEquals(0, results.size());
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/RowKeyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/RowKeyBuilder.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/RowKeyBuilder.java
index 5223ee6..1ce4906 100644
--- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/RowKeyBuilder.java
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/RowKeyBuilder.java
@@ -34,7 +34,7 @@ public interface RowKeyBuilder extends Serializable {
/**
* Build a row key for a given ProfileMeasurement.
- * <p>
+ *
* This method is useful when writing ProfileMeasurements to HBase.
*
* @param measurement The profile measurement.
@@ -46,15 +46,15 @@ public interface RowKeyBuilder extends Serializable {
/**
* Builds a list of row keys necessary to retrieve a profile's measurements over
* a time horizon.
- * <p>
+ *
* This method is useful when attempting to read ProfileMeasurements stored in HBase.
*
- * @param profile The name of the profile.
- * @param entity The name of the entity.
- * @param groups The group(s) used to sort the profile data.
- * @param durationAgo How long ago?
- * @param unit The time units of how long ago.
+ * @param profile The name of the profile.
+ * @param entity The name of the entity.
+ * @param groups The group(s) used to sort the profile data.
+ * @param start When the time horizon starts in epoch milliseconds.
+ * @param end When the time horizon ends in epoch milliseconds.
* @return All of the row keys necessary to retrieve the profile measurements.
*/
- List<byte[]> rowKeys(String profile, String entity, List<Object> groups, long durationAgo, TimeUnit unit);
+ List<byte[]> rowKeys(String profile, String entity, List<Object> groups, long start, long end);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java
index d1c48f8..3b12472 100644
--- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java
@@ -77,21 +77,21 @@ public class SaltyRowKeyBuilder implements RowKeyBuilder {
* @param profile The name of the profile.
* @param entity The name of the entity.
* @param groups The group(s) used to sort the profile data.
- * @param durationAgo How long ago?
- * @param unit The time units of how long ago.
+ * @param start When the time horizon starts in epoch milliseconds.
+ * @param end When the time horizon ends in epoch milliseconds.
* @return All of the row keys necessary to retrieve the profile measurements.
*/
@Override
- public List<byte[]> rowKeys(String profile, String entity, List<Object> groups, long durationAgo, TimeUnit unit) {
+ public List<byte[]> rowKeys(String profile, String entity, List<Object> groups, long start, long end) {
List<byte[]> rowKeys = new ArrayList<>();
- // find the time horizon
- long endTime = System.currentTimeMillis();
- long startTime = endTime - unit.toMillis(durationAgo);
+ // be forgiving of out-of-order start and end times; order is critical to this algorithm
+ end = Math.max(start, end);
+ start = Math.min(start, end);
// find the starting period and advance until the end time is reached
- ProfilePeriod period = new ProfilePeriod(startTime, periodDurationMillis, TimeUnit.MILLISECONDS);
- while(period.getStartTimeMillis() <= endTime) {
+ ProfilePeriod period = new ProfilePeriod(start, periodDurationMillis, TimeUnit.MILLISECONDS);
+ while(period.getStartTimeMillis() <= end) {
byte[] k = rowKey(profile, entity, period, groups);
rowKeys.add(k);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
index b4ebcf2..56f1d51 100644
--- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
+++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
@@ -219,7 +219,8 @@ public class SaltyRowKeyBuilderTest {
rowKeyBuilder = new SaltyRowKeyBuilder(saltDivisor, periodDuration, periodUnits);
// a dummy profile measurement
- long oldest = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(hoursAgo);
+ long now = System.currentTimeMillis();
+ long oldest = now - TimeUnit.HOURS.toMillis(hoursAgo);
ProfileMeasurement m = new ProfileMeasurement("profile", "entity", oldest, periodDuration, periodUnits);
m.setValue(22);
@@ -237,7 +238,7 @@ public class SaltyRowKeyBuilderTest {
}
// execute
- List<byte[]> actualKeys = rowKeyBuilder.rowKeys(measurement.getProfileName(), measurement.getEntity(), groups, hoursAgo, TimeUnit.HOURS);
+ List<byte[]> actualKeys = rowKeyBuilder.rowKeys(measurement.getProfileName(), measurement.getEntity(), groups, oldest, now);
// validate - expectedKeys == actualKeys
for(int i=0; i<actualKeys.size(); i++) {