You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2017/12/21 21:49:40 UTC
metron git commit: METRON-1365: Allow PROFILE_GET to return a default
value for a profile and entity that does not have a value written. closes
apache/incubator-metron#871
Repository: metron
Updated Branches:
refs/heads/master 196da12c4 -> 76bed5d75
METRON-1365: Allow PROFILE_GET to return a default value for a profile and entity that does not have a value written. closes apache/incubator-metron#871
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/76bed5d7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/76bed5d7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/76bed5d7
Branch: refs/heads/master
Commit: 76bed5d754fcf358809f0be7a034758b9b20fc5e
Parents: 196da12
Author: cstella <ce...@gmail.com>
Authored: Thu Dec 21 16:49:31 2017 -0500
Committer: cstella <ce...@gmail.com>
Committed: Thu Dec 21 16:49:31 2017 -0500
----------------------------------------------------------------------
.../metron-profiler-client/README.md | 1 +
.../profiler/client/HBaseProfilerClient.java | 35 +++++++++++------
.../metron/profiler/client/ProfilerClient.java | 10 +++--
.../profiler/client/stellar/GetProfile.java | 8 ++--
.../client/stellar/ProfilerClientConfig.java | 6 ++-
.../client/HBaseProfilerClientTest.java | 40 +++++++++++++-------
.../profiler/client/stellar/GetProfileTest.java | 38 +++++++++++++++++--
7 files changed, 102 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/76bed5d7/metron-analytics/metron-profiler-client/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/README.md b/metron-analytics/metron-profiler-client/README.md
index a15ccf8..4156058 100644
--- a/metron-analytics/metron-profiler-client/README.md
+++ b/metron-analytics/metron-profiler-client/README.md
@@ -58,6 +58,7 @@ want to change the global Client configuration so as not to disrupt the work of
| profiler.client.hbase.table | The name of the HBase table used to store profile data. | Optional | profiler |
| profiler.client.hbase.column.family | The name of the HBase column family used to store profile data. | Optional | P |
| profiler.client.salt.divisor | The salt divisor used to store profile data. | Optional | 1000 |
+| profiler.default.value | The default value to be returned if a profile is not written for a given period for a profile and entity. | Optional | null |
| hbase.provider.impl | The name of the HBaseTableProvider implementation class. | Optional | |
### Profile Selectors
http://git-wip-us.apache.org/repos/asf/metron/blob/76bed5d7/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
index 7c4ec84..de2d42c 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
@@ -33,6 +33,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -71,14 +72,15 @@ public class HBaseProfilerClient implements ProfilerClient {
* @param groups The groups used to sort the profile data.
* @param durationAgo How far in the past to fetch values from.
* @param unit The time unit of 'durationAgo'.
+ * @param defaultValue The default value to specify. If empty, the result will be sparse.
* @param <T> The type of values stored by the Profile.
* @return A list of values.
*/
@Override
- public <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long durationAgo, TimeUnit unit) {
+ public <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long durationAgo, TimeUnit unit, Optional<T> defaultValue) {
long end = System.currentTimeMillis();
long start = end - unit.toMillis(durationAgo);
- return fetch(clazz, profile, entity, groups, start, end);
+ return fetch(clazz, profile, entity, groups, start, end, defaultValue);
}
/**
@@ -90,11 +92,12 @@ public class HBaseProfilerClient implements ProfilerClient {
* @param groups The groups used to sort the profile data.
* @param start The start time in epoch milliseconds.
* @param end The end time in epoch milliseconds.
+ * @param defaultValue The default value to specify. If empty, the result will be sparse.
* @param <T> The type of values stored by the profile.
* @return A list of values.
*/
@Override
- public <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long start, long end) {
+ public <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long start, long end, Optional<T> defaultValue) {
byte[] columnFamily = Bytes.toBytes(columnBuilder.getColumnFamily());
byte[] columnQualifier = columnBuilder.getColumnQualifier("value");
@@ -108,7 +111,7 @@ public class HBaseProfilerClient implements ProfilerClient {
.collect(Collectors.toList());
// get the 'gets'
- return get(gets, columnQualifier, columnFamily, clazz);
+ return get(gets, columnQualifier, columnFamily, clazz, defaultValue);
}
/**
@@ -119,10 +122,11 @@ public class HBaseProfilerClient implements ProfilerClient {
* @param entity The name of the entity.
* @param groups The groups used to sort the profile data.
* @param periods The set of profile measurement periods
+ * @param defaultValue The default value to specify. If empty, the result will be sparse.
* @return A list of values.
*/
@Override
- public <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, Iterable<ProfilePeriod> periods) {
+ public <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, Iterable<ProfilePeriod> periods, Optional<T> defaultValue) {
byte[] columnFamily = Bytes.toBytes(columnBuilder.getColumnFamily());
byte[] columnQualifier = columnBuilder.getColumnQualifier("value");
@@ -136,7 +140,7 @@ public class HBaseProfilerClient implements ProfilerClient {
.collect(Collectors.toList());
// get the 'gets'
- return get(gets, columnQualifier, columnFamily, clazz);
+ return get(gets, columnQualifier, columnFamily, clazz, defaultValue);
}
/**
@@ -146,19 +150,26 @@ public class HBaseProfilerClient implements ProfilerClient {
* @param columnQualifier The column qualifier.
* @param columnFamily The column family.
* @param clazz The type expected in return.
+ * @param defaultValue The default value to specify. If empty, the result will be sparse.
* @param <T> The type expected in return.
* @return
*/
- private <T> List<T> get(List<Get> gets, byte[] columnQualifier, byte[] columnFamily, Class<T> clazz) {
+ private <T> List<T> get(List<Get> gets, byte[] columnQualifier, byte[] columnFamily, Class<T> clazz, Optional<T> defaultValue) {
List<T> values = new ArrayList<>();
try {
Result[] results = table.get(gets);
- Arrays.stream(results)
- .filter(r -> r.containsColumn(columnFamily, columnQualifier))
- .map(r -> r.getValue(columnFamily, columnQualifier))
- .forEach(val -> values.add(SerDeUtils.fromBytes(val, clazz)));
-
+ for(int i = 0;i < results.length;++i) {
+ Result result = results[i];
+ boolean exists = result.containsColumn(columnFamily, columnQualifier);
+ if(!exists && defaultValue.isPresent()) {
+ values.add(defaultValue.get());
+ }
+ else if(exists) {
+ byte[] val = result.getValue(columnFamily, columnQualifier);
+ values.add(SerDeUtils.fromBytes(val, clazz));
+ }
+ }
} catch(IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/76bed5d7/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
index 57b0e04..bab4ec9 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
@@ -23,6 +23,7 @@ package org.apache.metron.profiler.client;
import org.apache.metron.profiler.ProfilePeriod;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.TimeUnit;
/**
@@ -39,10 +40,11 @@ public interface ProfilerClient {
* @param groups The groups used to sort the profile data.
* @param durationAgo How far in the past to fetch values from.
* @param unit The time unit of 'durationAgo'.
+ * @param defaultValue The default value to specify. If empty, the result will be sparse.
* @param <T> The type of values stored by the Profile.
* @return A list of values.
*/
- <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long durationAgo, TimeUnit unit);
+ <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long durationAgo, TimeUnit unit, Optional<T> defaultValue);
/**
* Fetch the values stored in a profile based on a start and end timestamp.
@@ -53,10 +55,11 @@ public interface ProfilerClient {
* @param groups The groups used to sort the profile data.
* @param start The start time in epoch milliseconds.
* @param end The end time in epoch milliseconds.
+ * @param defaultValue The default value to specify. If empty, the result will be sparse.
* @param <T> The type of values stored by the profile.
* @return A list of values.
*/
- <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long start, long end);
+ <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, long start, long end, Optional<T> defaultValue);
/**
* Fetch the values stored in a profile based on a set of period keys.
@@ -66,8 +69,9 @@ public interface ProfilerClient {
* @param entity The name of the entity.
* @param groups The groups used to sort the profile data.
* @param periods The set of profile period keys
+ * @param defaultValue The default value to specify. If empty, the result will be sparse.
* @param <T> The type of values stored by the profile.
* @return A list of values.
*/
- <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, Iterable<ProfilePeriod> periods);
+ <T> List<T> fetch(Class<T> clazz, String profile, String entity, List<Object> groups, Iterable<ProfilePeriod> periods, Optional<T> defaultValue);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/76bed5d7/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
index 802c552..73cd5a1 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
@@ -163,7 +163,7 @@ public class GetProfile implements StellarFunction {
}
Map<String, Object> effectiveConfig = getEffectiveConfig(context, configOverridesMap);
-
+ Object defaultValue = null;
//lazily create new profiler client if needed
if (client == null || !cachedConfigMap.equals(effectiveConfig)) {
RowKeyBuilder rowKeyBuilder = getRowKeyBuilder(effectiveConfig);
@@ -172,8 +172,10 @@ public class GetProfile implements StellarFunction {
client = new HBaseProfilerClient(table, rowKeyBuilder, columnBuilder);
cachedConfigMap = effectiveConfig;
}
-
- return client.fetch(Object.class, profile, entity, groups, periods.orElse(new ArrayList<>(0)));
+ if(cachedConfigMap != null) {
+ defaultValue = ProfilerClientConfig.PROFILER_DEFAULT_VALUE.get(cachedConfigMap);
+ }
+ return client.fetch(Object.class, profile, entity, groups, periods.orElse(new ArrayList<>(0)), Optional.ofNullable(defaultValue));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/76bed5d7/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerClientConfig.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerClientConfig.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerClientConfig.java
index 351b807..9bbc29d 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerClientConfig.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerClientConfig.java
@@ -56,8 +56,12 @@ public enum ProfilerClientConfig {
/**
* A global property that defines the salt divisor used to store profile data.
*/
- PROFILER_SALT_DIVISOR("profiler.client.salt.divisor", 1000L, Long.class);
+ PROFILER_SALT_DIVISOR("profiler.client.salt.divisor", 1000L, Long.class),
+ /**
+ * The default value to be returned if a profile is not written for a given period for a profile and entity.
+ */
+ PROFILER_DEFAULT_VALUE("profiler.default.value", null, Object.class);
String key;
Object defaultValue;
Class<?> valueType;
http://git-wip-us.apache.org/repos/asf/metron/blob/76bed5d7/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
index 8519f10..96c0d91 100644
--- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
@@ -34,9 +34,11 @@ import org.junit.Test;
import java.util.Arrays;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Tests the HBaseProfilerClient.
@@ -99,12 +101,15 @@ public class HBaseProfilerClientTest {
profileWriter.write(m, count, Arrays.asList("weekdays"), val -> expectedValue);
profileWriter.write(m, count, Arrays.asList("weekends"), val -> 0);
- // execute
- List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", Arrays.asList("weekdays"), hours, TimeUnit.HOURS);
+ //valid results
+ {
+ // execute
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", Arrays.asList("weekdays"), hours, TimeUnit.HOURS, Optional.empty());
- // validate
- assertEquals(count, results.size());
- results.forEach(actual -> assertEquals(expectedValue, (int) actual));
+ // validate
+ assertEquals(count, results.size());
+ results.forEach(actual -> assertEquals(expectedValue, (int) actual));
+ }
}
/**
@@ -128,10 +133,19 @@ public class HBaseProfilerClientTest {
// execute
List<Object> doesNotExist = Arrays.asList("does-not-exist");
- List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", doesNotExist, hours, TimeUnit.HOURS);
-
- // validate
- assertEquals(0, results.size());
+ {
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", doesNotExist, hours, TimeUnit.HOURS, Optional.empty());
+
+ // validate
+ assertEquals(0, results.size());
+ }
+ {
+ //with a default value, we'd expect a bunch of 0's
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", doesNotExist, hours, TimeUnit.HOURS, Optional.of(0));
+ //8 or 9 15 minute periods in 2 hours depending on when you start
+ assertTrue(results.size() == 8 || results.size() == 9);
+ results.forEach(actual -> assertEquals(0, (int) actual));
+ }
}
/**
@@ -152,7 +166,7 @@ public class HBaseProfilerClientTest {
profileWriter.write(m, hours * periodsPerHour, group, val -> 1000);
// execute
- List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", group, 2, TimeUnit.MILLISECONDS);
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", group, 2, TimeUnit.MILLISECONDS, Optional.empty());
// validate - there should NOT be any results from just 2 milliseconds ago
assertEquals(0, results.size());
@@ -179,7 +193,7 @@ public class HBaseProfilerClientTest {
profileWriter.write(m, count, Arrays.asList("weekends"), val -> 0);
// execute
- List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", Arrays.asList("weekdays"), startTime, endTime);
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", Arrays.asList("weekdays"), startTime, endTime, Optional.empty());
// validate
assertEquals(count, results.size());
@@ -210,7 +224,7 @@ public class HBaseProfilerClientTest {
// execute
List<Object> doesNotExist = Arrays.asList("does-not-exist");
- List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", doesNotExist, startTime, endTime);
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", doesNotExist, startTime, endTime, Optional.empty());
// validate
assertEquals(0, results.size());
@@ -238,7 +252,7 @@ public class HBaseProfilerClientTest {
// execute
final long endFetchAt = System.currentTimeMillis();
final long startFetchAt = endFetchAt - TimeUnit.MILLISECONDS.toMillis(30);
- List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", group, startFetchAt, endFetchAt);
+ List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", group, startFetchAt, endFetchAt, Optional.empty());
// validate - there should NOT be any results from just 2 milliseconds ago
assertEquals(0, results.size());
http://git-wip-us.apache.org/repos/asf/metron/blob/76bed5d7/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java
index 307b548..6ed6e64 100644
--- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java
@@ -39,12 +39,13 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.Arrays;
-import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+import java.util.Map;
import java.util.HashMap;
+import java.util.Collections;
+import java.util.Arrays;
+import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.*;
@@ -311,6 +312,35 @@ public class GetProfileTest {
}
/**
+ * Default value should be able to be specified
+ */
+ @Test
+ public void testWithDefaultValue() {
+ String expr = "PROFILE_GET('profile1', 'entity1', PROFILE_FIXED(4, 'HOURS'))";
+ @SuppressWarnings("unchecked")
+ List<Integer> result = run(expr, List.class);
+
+ // validate - expect to fail to read any values because we didn't write any.
+ Assert.assertEquals(0, result.size());
+
+ // execute - read the profile values - with config_override.
+ // first two override values are strings, third is deliberately a number.
+ testOverride("{'profiler.default.value' : 0}", 0);
+ testOverride("{'profiler.default.value' : 'metron'}", "metron");
+ testOverride("{'profiler.default.value' : []}", new ArrayList<>());
+ }
+
+ private void testOverride(String overrides, Object defaultVal) {
+ String expr = "PROFILE_GET('profile1', 'entity1', PROFILE_FIXED(4, 'HOURS'), [], " + overrides + ")";
+ List<Object> result = run(expr, List.class);
+
+ // validate - expect to read all values from the past 4 hours (16 or 17 values depending on start time)
+ // but they should all be the default value.
+ Assert.assertTrue(result.size() == 16 || result.size() == 17);
+ result.forEach(actual -> Assert.assertEquals(defaultVal, actual));
+ }
+
+ /**
* Values should be retrievable that were written with configuration different than current global config.
*/
@Test