You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2018/08/20 21:54:32 UTC
metron git commit: METRON-1705 Create ProfilePeriod Using Period ID
(nickwallen) closes apache/metron#1148
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1699-create-batch-profiler 401bd7a18 -> 6fb50a18d
METRON-1705 Create ProfilePeriod Using Period ID (nickwallen) closes apache/metron#1148
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6fb50a18
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6fb50a18
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6fb50a18
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 6fb50a18d75c3694c75596ab45bcc7531711eec7
Parents: 401bd7a
Author: nickwallen <ni...@nickallen.org>
Authored: Mon Aug 20 17:53:58 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Mon Aug 20 17:53:58 2018 -0400
----------------------------------------------------------------------
.../metron/profiler/DefaultProfileBuilder.java | 4 +-
.../metron/profiler/ProfileMeasurement.java | 2 +-
.../apache/metron/profiler/ProfilePeriod.java | 43 ++++++++++++----
.../profiler/DefaultProfileBuilderTest.java | 6 +--
.../metron/profiler/ProfilePeriodTest.java | 52 +++++++++++++++-----
5 files changed, 79 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/6fb50a18/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
index 66034ac..ced1e8b 100644
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
+++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
@@ -160,10 +160,8 @@ public class DefaultProfileBuilder implements ProfileBuilder, Serializable {
*/
@Override
public Optional<ProfileMeasurement> flush() {
-
Optional<ProfileMeasurement> result;
- ProfilePeriod period = new ProfilePeriod(maxTimestamp, periodDurationMillis, TimeUnit.MILLISECONDS);
-
+ ProfilePeriod period = ProfilePeriod.fromTimestamp(maxTimestamp, periodDurationMillis, TimeUnit.MILLISECONDS);
try {
// execute the 'profile' expression
String profileExpression = definition
http://git-wip-us.apache.org/repos/asf/metron/blob/6fb50a18/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
index 4737c3d..4d24fac 100644
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
+++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
@@ -95,7 +95,7 @@ public class ProfileMeasurement implements Serializable {
}
public ProfileMeasurement withPeriod(long whenMillis, long periodDuration, TimeUnit periodUnits) {
- this.withPeriod(new ProfilePeriod(whenMillis, periodDuration, periodUnits));
+ this.withPeriod(ProfilePeriod.fromTimestamp(whenMillis, periodDuration, periodUnits));
return this;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/6fb50a18/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
index cbb8275..78aa796 100644
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
+++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
@@ -47,19 +47,46 @@ public class ProfilePeriod implements Serializable {
*/
private long durationMillis;
+ /**
+ * @param periodId A monotonically increasing number identifying the period.
+ * @param duration The duration of each profile period.
+ * @param units The units of the duration; hours, minutes, etc.
+ */
+ private ProfilePeriod(long periodId, long duration, TimeUnit units) {
+ this.durationMillis = units.toMillis(duration);
+ this.period = periodId;
+ }
/**
+ * Creates a {@link ProfilePeriod} given a timestamp defined in milliseconds
+ * from the epoch.
+ *
* @param epochMillis A timestamp contained somewhere within the profile period.
* @param duration The duration of each profile period.
* @param units The units of the duration; hours, minutes, etc.
*/
- public ProfilePeriod(long epochMillis, long duration, TimeUnit units) {
+ public static ProfilePeriod fromTimestamp(long epochMillis, long duration, TimeUnit units) {
if(duration <= 0) {
- throw new IllegalArgumentException(format(
- "period duration must be greater than 0; got '%d %s'", duration, units));
+ throw new IllegalArgumentException(format("period duration must be > 0; got '%d %s'", duration, units));
}
- this.durationMillis = units.toMillis(duration);
- this.period = epochMillis / durationMillis;
+ long durationMillis = units.toMillis(duration);
+ long periodId = epochMillis / durationMillis;
+ return new ProfilePeriod(periodId, duration, units);
+ }
+
+ /**
+ * Creates a {@link ProfilePeriod} given a timestamp defined in milliseconds
+ * from the epoch.
+ *
+ * @param periodId A monotonically increasing number identifying the period.
+ * @param duration The duration of each profile period.
+ * @param units The units of the duration; hours, minutes, etc.
+ */
+ public static ProfilePeriod fromPeriodId(long periodId, long duration, TimeUnit units) {
+ if(periodId < 0) {
+ throw new IllegalArgumentException(format("period id must be >= 0; got '%d'", periodId));
+ }
+ return new ProfilePeriod(periodId, duration, units);
}
/**
@@ -80,15 +107,13 @@ public class ProfilePeriod implements Serializable {
* Returns the next ProfilePeriod in time.
*/
public ProfilePeriod next() {
- long nextStart = getStartTimeMillis() + durationMillis;
- return new ProfilePeriod(nextStart, durationMillis, TimeUnit.MILLISECONDS);
+ return fromPeriodId(period + 1, durationMillis, TimeUnit.MILLISECONDS);
}
public long getPeriod() {
return period;
}
-
public long getDurationMillis() {
return durationMillis;
}
@@ -126,7 +151,7 @@ public class ProfilePeriod implements Serializable {
, Function<ProfilePeriod,T> transformation
)
{
- ProfilePeriod period = new ProfilePeriod(startEpochMillis, duration, units);
+ ProfilePeriod period = ProfilePeriod.fromTimestamp(startEpochMillis, duration, units);
List<T> ret = new ArrayList<>();
while(period.getStartTimeMillis() <= endEpochMillis) {
if(!inclusionPredicate.isPresent() || inclusionPredicate.get().test(period)) {
http://git-wip-us.apache.org/repos/asf/metron/blob/6fb50a18/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java
index 24eb5f8..3d0b4cc 100644
--- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java
+++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java
@@ -235,7 +235,7 @@ public class DefaultProfileBuilderTest {
assertTrue(m.isPresent());
// validate the profile period
- ProfilePeriod expected = new ProfilePeriod(timestamp, 10, TimeUnit.MINUTES);
+ ProfilePeriod expected = ProfilePeriod.fromTimestamp(timestamp, 10, TimeUnit.MINUTES);
assertEquals(expected, m.get().getPeriod());
}
{
@@ -248,7 +248,7 @@ public class DefaultProfileBuilderTest {
assertTrue(m.isPresent());
// validate the profile period
- ProfilePeriod expected = new ProfilePeriod(timestamp, 10, TimeUnit.MINUTES);
+ ProfilePeriod expected = ProfilePeriod.fromTimestamp(timestamp, 10, TimeUnit.MINUTES);
assertEquals(expected, m.get().getPeriod());
}
}
@@ -314,7 +314,7 @@ public class DefaultProfileBuilderTest {
// setup
long timestamp = 1503081070340L;
- ProfilePeriod period = new ProfilePeriod(timestamp, 10, TimeUnit.MINUTES);
+ ProfilePeriod period = ProfilePeriod.fromTimestamp(timestamp, 10, TimeUnit.MINUTES);
definition = JSONUtils.INSTANCE.load(testStateAvailableToGroupBy, ProfileConfig.class);
builder = new DefaultProfileBuilder.Builder()
.withDefinition(definition)
http://git-wip-us.apache.org/repos/asf/metron/blob/6fb50a18/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfilePeriodTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfilePeriodTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfilePeriodTest.java
index f52bd09..295117a 100644
--- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfilePeriodTest.java
+++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfilePeriodTest.java
@@ -48,7 +48,7 @@ public class ProfilePeriodTest {
long duration = 1;
TimeUnit units = TimeUnit.HOURS;
- ProfilePeriod period = new ProfilePeriod(0, duration, units);
+ ProfilePeriod period = ProfilePeriod.fromTimestamp(0, duration, units);
assertEquals(0, period.getPeriod());
assertEquals(0, period.getStartTimeMillis());
assertEquals(units.toMillis(duration), period.getDurationMillis());
@@ -59,7 +59,7 @@ public class ProfilePeriodTest {
long duration = 1;
TimeUnit units = TimeUnit.MINUTES;
- ProfilePeriod period = new ProfilePeriod(AUG2016, duration, units);
+ ProfilePeriod period = ProfilePeriod.fromTimestamp(AUG2016, duration, units);
assertEquals(24535527, period.getPeriod());
assertEquals(1472131620000L, period.getStartTimeMillis()); // Thu, 25 Aug 2016 13:27:00 GMT
assertEquals(units.toMillis(duration), period.getDurationMillis());
@@ -70,7 +70,7 @@ public class ProfilePeriodTest {
long duration = 15;
TimeUnit units = TimeUnit.MINUTES;
- ProfilePeriod period = new ProfilePeriod(AUG2016, duration, units);
+ ProfilePeriod period = ProfilePeriod.fromTimestamp(AUG2016, duration, units);
assertEquals(1635701, period.getPeriod());
assertEquals(1472130900000L, period.getStartTimeMillis()); // Thu, 25 Aug 2016 13:15:00 GMT
assertEquals(units.toMillis(duration), period.getDurationMillis());
@@ -81,7 +81,7 @@ public class ProfilePeriodTest {
long duration = 1;
TimeUnit units = TimeUnit.HOURS;
- ProfilePeriod period = new ProfilePeriod(AUG2016, duration, units);
+ ProfilePeriod period = ProfilePeriod.fromTimestamp(AUG2016, duration, units);
assertEquals(408925, period.getPeriod());
assertEquals(1472130000000L, period.getStartTimeMillis()); // Thu, 25 Aug 2016 13:00:00 GMT
assertEquals(units.toMillis(duration), period.getDurationMillis());
@@ -92,7 +92,7 @@ public class ProfilePeriodTest {
long duration = 2;
TimeUnit units = TimeUnit.HOURS;
- ProfilePeriod period = new ProfilePeriod(AUG2016, duration, units);
+ ProfilePeriod period = ProfilePeriod.fromTimestamp(AUG2016, duration, units);
assertEquals(204462, period.getPeriod());
assertEquals(1472126400000L, period.getStartTimeMillis()); // Thu, 25 Aug 2016 12:00:00 GMT
assertEquals(units.toMillis(duration), period.getDurationMillis());
@@ -103,7 +103,7 @@ public class ProfilePeriodTest {
long duration = 8;
TimeUnit units = TimeUnit.HOURS;
- ProfilePeriod period = new ProfilePeriod(AUG2016, duration, units);
+ ProfilePeriod period = ProfilePeriod.fromTimestamp(AUG2016, duration, units);
assertEquals(51115, period.getPeriod());
assertEquals(1472112000000L, period.getStartTimeMillis()); // Thu, 25 Aug 2016 08:00:00 GMT
assertEquals(units.toMillis(duration), period.getDurationMillis());
@@ -114,7 +114,7 @@ public class ProfilePeriodTest {
long duration = 15;
TimeUnit units = TimeUnit.MINUTES;
- ProfilePeriod previous = new ProfilePeriod(AUG2016, duration, units);
+ ProfilePeriod previous = ProfilePeriod.fromTimestamp(AUG2016, duration, units);
IntStream.range(0, 100).forEach(i -> {
ProfilePeriod next = previous.next();
@@ -128,7 +128,7 @@ public class ProfilePeriodTest {
public void testPeriodDurationOfZero() {
long duration = 0;
TimeUnit units = TimeUnit.HOURS;
- new ProfilePeriod(0, duration, units);
+ ProfilePeriod.fromTimestamp(0, duration, units);
}
/**
@@ -137,8 +137,7 @@ public class ProfilePeriodTest {
*/
@Test
public void testKryoSerialization() throws Exception {
-
- ProfilePeriod expected = new ProfilePeriod(AUG2016, 1, TimeUnit.HOURS);
+ ProfilePeriod expected = ProfilePeriod.fromTimestamp(AUG2016, 1, TimeUnit.HOURS);
// round-trip java serialization
byte[] raw = SerDeUtils.toBytes(expected);
@@ -154,8 +153,7 @@ public class ProfilePeriodTest {
*/
@Test
public void testJavaSerialization() throws Exception {
-
- ProfilePeriod expected = new ProfilePeriod(AUG2016, 1, TimeUnit.HOURS);
+ ProfilePeriod expected = ProfilePeriod.fromTimestamp(AUG2016, 1, TimeUnit.HOURS);
// serialize using java
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@@ -173,4 +171,34 @@ public class ProfilePeriodTest {
// ensure that the round-trip was successful
assertEquals(expected, actual);
}
+
+ /**
+ * A {@link ProfilePeriod} can also be created from the period identifier and duration.
+ */
+ @Test
+ public void testFromPeriodId() {
+ ProfilePeriod expected = ProfilePeriod.fromTimestamp(AUG2016, 1, TimeUnit.HOURS);
+
+ // create the same period, but use the period identifier and duration
+ long periodId = expected.getPeriod();
+ long duration = expected.getDurationMillis();
+ ProfilePeriod actual = ProfilePeriod.fromPeriodId(periodId, duration, TimeUnit.MILLISECONDS);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testWithNegativePeriodId() {
+ ProfilePeriod.fromPeriodId(-1, 1, TimeUnit.HOURS);
+ }
+
+ /**
+ * The first period identifier 0 should start at the epoch.
+ */
+ @Test
+ public void testFromPeriodIdAtEpoch() {
+ assertEquals(
+ ProfilePeriod.fromTimestamp(0, 1, TimeUnit.HOURS),
+ ProfilePeriod.fromPeriodId(0, 1, TimeUnit.HOURS));
+ }
}