You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/08/24 18:47:08 UTC
incubator-metron git commit: METRON-377 Enable Profiles that Use
Non-Single Pass Summary Functions (nickwallen) closes
apache/incubator-metron#226
Repository: incubator-metron
Updated Branches:
refs/heads/master fa946169c -> 86ce763e5
METRON-377 Enable Profiles that Use Non-Single Pass Summary Functions (nickwallen) closes apache/incubator-metron#226
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/86ce763e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/86ce763e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/86ce763e
Branch: refs/heads/master
Commit: 86ce763e55e17f50a6ea8a92661a9b0acc178fe3
Parents: fa94616
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Aug 24 14:45:51 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Wed Aug 24 14:45:51 2016 -0400
----------------------------------------------------------------------
.../config/zookeeper/percentiles/profiler.json | 13 +
.../integration/ProfilerIntegrationTest.java | 18 ++
.../metron/common/dsl/StellarFunctions.java | 34 +--
.../common/dsl/functions/StellarStatistics.java | 199 ++++++++++++++
.../functions/StellarStatisticsFunctions.java | 270 +++++++++++++++++++
.../functions/SummaryStatisticsFunctions.java | 231 ----------------
.../StellarStatisticalFunctionsTest.java | 184 -------------
.../stellar/StellarStatisticsFunctionsTest.java | 270 +++++++++++++++++++
8 files changed, 788 insertions(+), 431 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/86ce763e/metron-analytics/metron-profiler/src/test/config/zookeeper/percentiles/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/percentiles/profiler.json b/metron-analytics/metron-profiler/src/test/config/zookeeper/percentiles/profiler.json
new file mode 100644
index 0000000..8ef4352
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/test/config/zookeeper/percentiles/profiler.json
@@ -0,0 +1,13 @@
+{
+ "inputTopic": "indexing",
+ "profiles": [
+ {
+ "profile": "percentiles",
+ "foreach": "ip_src_addr",
+ "onlyif": "protocol == 'HTTP'",
+ "init": { "s": "STATS_INIT(100)" },
+ "update": { "s": "STATS_ADD(s, length)" },
+ "result": "STATS_PERCENTILE(s, 0.7)"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/86ce763e/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
index b652afc..9513305 100644
--- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
@@ -197,6 +197,24 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
Assert.assertEquals(10.0, actual, 0.01);
}
+ @Test
+ public void testPercentiles() throws Exception {
+
+ setup(TEST_RESOURCES + "/config/zookeeper/percentiles");
+
+ // start the topology and write test messages to kafka
+ fluxComponent.submitTopology();
+ kafkaComponent.writeMessages(Constants.INDEXING_TOPIC, input);
+
+ // verify - ensure the profile is being persisted
+ waitOrTimeout(() -> profilerTable.getPutLog().size() > 0,
+ timeout(seconds(90)));
+
+ // verify - the 70th percentile of 5 x 20s = 20.0
+ double actual = readDouble(ProfileHBaseMapper.QVALUE);
+ Assert.assertEquals(20.0, actual, 0.01);
+ }
+
/**
* Reads a Double value written by the Profiler.
* @param columnQual The column qualifier.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/86ce763e/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java
index 36c361a..1be7955 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java
@@ -25,7 +25,7 @@ import org.apache.metron.common.dsl.functions.MaaSFunctions;
import org.apache.metron.common.dsl.functions.MapFunctions;
import org.apache.metron.common.dsl.functions.NetworkFunctions;
import org.apache.metron.common.dsl.functions.StringFunctions;
-import org.apache.metron.common.dsl.functions.SummaryStatisticsFunctions;
+import org.apache.metron.common.dsl.functions.StellarStatisticsFunctions;
import org.apache.metron.common.field.transformation.IPProtocolTransformation;
import org.apache.metron.common.field.validation.network.DomainValidation;
import org.apache.metron.common.field.validation.network.EmailValidation;
@@ -89,21 +89,23 @@ public enum StellarFunctions implements StellarFunction {
MODEL_APPLY(new MaaSFunctions.ModelApply()),
// summary statistics
- STATS_INIT(new SummaryStatisticsFunctions.Init()),
- STATS_ADD(new SummaryStatisticsFunctions.Add()),
- STATS_COUNT(new SummaryStatisticsFunctions.Count()),
- STATS_MEAN(new SummaryStatisticsFunctions.Mean()),
- STATS_GEOMETRIC_MEAN(new SummaryStatisticsFunctions.GeometricMean()),
- STATS_MAX(new SummaryStatisticsFunctions.Max()),
- STATS_MIN(new SummaryStatisticsFunctions.Min()),
- STATS_SUM(new SummaryStatisticsFunctions.Sum()),
- STATS_POPULATION_VARIANCE(new SummaryStatisticsFunctions.PopulationVariance()),
- STATS_VARIANCE(new SummaryStatisticsFunctions.Variance()),
- STATS_SECOND_MOMENT(new SummaryStatisticsFunctions.SecondMoment()),
- STATS_QUADRATIC_MEAN(new SummaryStatisticsFunctions.QuadraticMean()),
- STATS_SD(new SummaryStatisticsFunctions.StandardDeviation()),
- STATS_SUM_LOGS(new SummaryStatisticsFunctions.SumLogs()),
- STATS_SUM_SQUARES(new SummaryStatisticsFunctions.SumSquares());
+ STATS_INIT(new StellarStatisticsFunctions.Init()),
+ STATS_ADD(new StellarStatisticsFunctions.Add()),
+ STATS_COUNT(new StellarStatisticsFunctions.Count()),
+ STATS_MEAN(new StellarStatisticsFunctions.Mean()),
+ STATS_GEOMETRIC_MEAN(new StellarStatisticsFunctions.GeometricMean()),
+ STATS_MAX(new StellarStatisticsFunctions.Max()),
+ STATS_MIN(new StellarStatisticsFunctions.Min()),
+ STATS_SUM(new StellarStatisticsFunctions.Sum()),
+ STATS_POPULATION_VARIANCE(new StellarStatisticsFunctions.PopulationVariance()),
+ STATS_VARIANCE(new StellarStatisticsFunctions.Variance()),
+ STATS_QUADRATIC_MEAN(new StellarStatisticsFunctions.QuadraticMean()),
+ STATS_SD(new StellarStatisticsFunctions.StandardDeviation()),
+ STATS_SUM_LOGS(new StellarStatisticsFunctions.SumLogs()),
+ STATS_SUM_SQUARES(new StellarStatisticsFunctions.SumSquares()),
+ STATS_KURTOSIS(new StellarStatisticsFunctions.Kurtosis()),
+ STATS_SKEWNESS(new StellarStatisticsFunctions.Skewness()),
+ STATS_PERCENTILE(new StellarStatisticsFunctions.Percentile());
private static class Predicate2Transformation extends BaseStellarFunction {
Predicate<List<Object>> pred;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/86ce763e/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatistics.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatistics.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatistics.java
new file mode 100644
index 0000000..b199c36
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatistics.java
@@ -0,0 +1,199 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.common.dsl.functions;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+
+import java.io.Serializable;
+
+/**
+ * Provides basic summary statistics to Stellar.
+ *
+ * Used as an adapter to provide a single interface to two underlying Commons
+ * Math classes that provide summary statistics.
+ */
+public class StellarStatistics implements Serializable {
+
+ /**
+ * DescriptiveStatistics stores a rolling window of input data elements
+ * which is then used to calculate the summary statistic. There are some
+ * summary statistics like kurtosis and percentiles, that can only
+ * be calculated with this.
+ *
+ * This implementation is used if the windowSize > 0.
+ */
+ private DescriptiveStatistics descStats;
+
+ /**
+ * SummaryStatistics can be used for summary statistics that only
+ * require a single pass over the input data. This has the advantage
+ * of being less memory intensive, but not all summary statistics are
+ * available.
+ *
+ * This implementation is used if the windowSize == 0.
+ */
+ private SummaryStatistics summStats;
+
+ /**
+ * @param windowSize The number of input data elements to maintain in memory. If
+ * windowSize == 0, then no data elements will be maintained in
+ * memory.
+ */
+ public StellarStatistics(int windowSize) {
+
+ // only one of the underlying implementation classes will be used at a time
+ if(windowSize > 0) {
+ descStats = new DescriptiveStatistics(windowSize);
+ } else {
+ summStats = new SummaryStatistics();
+ }
+ }
+
+ public void addValue(double value) {
+ if(descStats != null) {
+ descStats.addValue(value);
+ } else {
+ summStats.addValue(value);
+ }
+ }
+
+ public long getCount() {
+ if(descStats != null) {
+ return descStats.getN();
+ } else {
+ return summStats.getN();
+ }
+ }
+
+ public double getMin() {
+ if(descStats != null) {
+ return descStats.getMin();
+ } else {
+ return summStats.getMin();
+ }
+ }
+
+ public double getMax() {
+ if(descStats != null) {
+ return descStats.getMax();
+ } else {
+ return summStats.getMax();
+ }
+ }
+
+ public double getMean() {
+ if(descStats != null) {
+ return descStats.getMean();
+ } else {
+ return summStats.getMean();
+ }
+ }
+
+ public double getSum() {
+ if(descStats != null) {
+ return descStats.getSum();
+ } else {
+ return summStats.getSum();
+ }
+ }
+
+ public double getVariance() {
+ if(descStats != null) {
+ return descStats.getVariance();
+ } else {
+ return summStats.getVariance();
+ }
+ }
+
+ public double getStandardDeviation() {
+ if(descStats != null) {
+ return descStats.getStandardDeviation();
+ } else {
+ return summStats.getStandardDeviation();
+ }
+ }
+
+ public double getGeometricMean() {
+ if(descStats != null) {
+ return descStats.getGeometricMean();
+ } else {
+ return summStats.getGeometricMean();
+ }
+ }
+
+ public double getPopulationVariance() {
+ if(descStats != null) {
+ return descStats.getPopulationVariance();
+ } else {
+ return summStats.getPopulationVariance();
+ }
+ }
+
+ public double getQuadraticMean() {
+ if(descStats != null) {
+ return descStats.getQuadraticMean();
+ } else {
+ return summStats.getQuadraticMean();
+ }
+ }
+
+ public double getSumLogs() {
+ if(descStats != null) {
+ throw new NotImplementedException("sum logs not available if 'windowSize' > 0");
+ } else {
+ return summStats.getSumOfLogs();
+ }
+ }
+
+ public double getSumSquares() {
+ if(descStats != null) {
+ return descStats.getSumsq();
+ } else {
+ return summStats.getSumsq();
+ }
+ }
+
+ public double getKurtosis() {
+ if(descStats != null) {
+ return descStats.getKurtosis();
+ } else {
+ throw new NotImplementedException("kurtosis not available if 'windowSize' = 0");
+ }
+ }
+
+ public double getSkewness() {
+ if(descStats != null) {
+ return descStats.getSkewness();
+ } else {
+ throw new NotImplementedException("skewness not available if 'windowSize' = 0");
+ }
+ }
+
+ public double getPercentile(double p) {
+ if(descStats != null) {
+ return descStats.getPercentile(p);
+ } else {
+ throw new NotImplementedException("percentile not available if 'windowSize' = 0");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/86ce763e/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatisticsFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatisticsFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatisticsFunctions.java
new file mode 100644
index 0000000..adf439b
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatisticsFunctions.java
@@ -0,0 +1,270 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.common.dsl.functions;
+
+import org.apache.metron.common.dsl.BaseStellarFunction;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.metron.common.utils.ConversionUtils.convert;
+
+/**
+ * Provides Stellar functions that can calculate summary statistics on
+ * streams of data.
+ */
+public class StellarStatisticsFunctions {
+
+ /**
+ * Initializes the summary statistics.
+ *
+ * Initialization can occur from either STATS_INIT and STATS_ADD.
+ */
+ private static StellarStatistics statsInit(List<Object> args) {
+ int windowSize = 0;
+ if(args.size() > 0 && args.get(0) instanceof Number) {
+ windowSize = convert(args.get(0), Integer.class);
+ }
+
+ return new StellarStatistics(windowSize);
+ }
+
+ /**
+ * Initialize the summary statistics.
+ *
+ * STATS_INIT (window_size)
+ *
+ * window_size The number of input data values to maintain in a rolling window
+ * in memory. If equal to 0, then no rolling window is maintained.
+ * Using no rolling window is less memory intensive, but cannot
+ * calculate certain statistics like percentiles and kurtosis.
+ */
+ public static class Init extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ return statsInit(args);
+ }
+ }
+
+ /**
+ * Add an input value to those that are used to calculate the summary statistics.
+ *
+ * STATS_ADD (stats, value [, value2, value3, ...])
+ */
+ public static class Add extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+
+ // initialize a stats object, if one does not already exist
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ if(stats == null) {
+ stats = statsInit(Collections.emptyList());
+ }
+
+ // add each of the numeric values
+ for(int i=1; i<args.size(); i++) {
+ double value = convert(args.get(i), Double.class);
+ stats.addValue(value);
+ }
+
+ return stats;
+ }
+ }
+
+
+ /**
+ * Calculates the mean.
+ *
+ * STATS_MEAN (stats)
+ */
+ public static class Mean extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getMean() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the geometric mean.
+ */
+ public static class GeometricMean extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getGeometricMean() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the sum.
+ */
+ public static class Sum extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getSum() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the max.
+ */
+ public static class Max extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getMax() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the min.
+ */
+ public static class Min extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getMin() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the count of elements
+ */
+ public static class Count extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? convert(stats.getCount(), Double.class) : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the population variance.
+ */
+ public static class PopulationVariance extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getPopulationVariance() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the variance.
+ */
+ public static class Variance extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getVariance() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the quadratic mean.
+ */
+ public static class QuadraticMean extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getQuadraticMean() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the standard deviation.
+ */
+ public static class StandardDeviation extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getStandardDeviation() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the sum of logs.
+ */
+ public static class SumLogs extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getSumLogs() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the sum of squares.
+ */
+ public static class SumSquares extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getSumSquares() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the kurtosis.
+ */
+ public static class Kurtosis extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getKurtosis() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the skewness.
+ */
+ public static class Skewness extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ return (stats != null) ? stats.getSkewness() : Double.NaN;
+ }
+ }
+
+ /**
+ * Calculates the Pth percentile.
+ *
+ * STATS_PERCENTILE(stats, 0.90)
+ */
+ public static class Percentile extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ StellarStatistics stats = convert(args.get(0), StellarStatistics.class);
+ Double p = convert(args.get(1), Double.class);
+
+ Double result;
+ if(stats == null || p == null) {
+ result = Double.NaN;
+ } else {
+ result = stats.getPercentile(p);
+ }
+
+ return result;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/86ce763e/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/SummaryStatisticsFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/SummaryStatisticsFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/SummaryStatisticsFunctions.java
deleted file mode 100644
index d8782d6..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/SummaryStatisticsFunctions.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.metron.common.dsl.functions;
-
-import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
-import org.apache.metron.common.dsl.BaseStellarFunction;
-
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.metron.common.utils.ConversionUtils.convert;
-
-/**
- * Provides Stellar functions that can calculate summary statistics on
- * streams of data.
- *
- * These functions are limited to those that can be calculated in a
- * single-pass so that the values are not stored in memory. This leverages
- * the commons-math SummaryStatistics class.
- */
-public class SummaryStatisticsFunctions {
-
- /**
- * Initializes the summary statistics.
- *
- * Initialization can occur from either STATS_INIT and STATS_ADD.
- */
- private static SummaryStatistics statsInit(List<Object> args) {
- return new SummaryStatistics();
- }
-
- /**
- * Initialize the summary statistics.
- *
- * STATS_INIT ()
- */
- public static class Init extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- return statsInit(args);
- }
- }
-
- /**
- * Add an input value to those that are used to calculate the summary statistics.
- *
- * STATS_ADD (stats, value [, value2, value3, ...])
- */
- public static class Add extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
-
- // initialize a stats object, if one does not already exist
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- if(stats == null) {
- stats = statsInit(Collections.emptyList());
- }
-
- // add each of the numeric values
- for(int i=1; i<args.size(); i++) {
- double value = convert(args.get(i), Double.class);
- stats.addValue(value);
- }
-
- return stats;
- }
- }
-
- /**
- * Calculates the mean.
- *
- * STATS_MEAN (stats)
- */
- public static class Mean extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getMean() : Double.NaN;
- }
- }
-
- /**
- * Calculates the geometric mean.
- */
- public static class GeometricMean extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getGeometricMean() : Double.NaN;
- }
- }
-
- /**
- * Calculates the sum.
- */
- public static class Sum extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getSum() : Double.NaN;
- }
- }
-
- /**
- * Calculates the max.
- */
- public static class Max extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getMax() : Double.NaN;
- }
- }
-
- /**
- * Calculates the min.
- */
- public static class Min extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getMin() : Double.NaN;
- }
- }
-
- /**
- * Calculates the count of elements
- */
- public static class Count extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? convert(stats.getN(), Double.class) : Double.NaN;
- }
- }
-
- /**
- * Calculates the population variance.
- */
- public static class PopulationVariance extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getPopulationVariance() : Double.NaN;
- }
- }
-
- /**
- * Calculates the variance.
- */
- public static class Variance extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getVariance() : Double.NaN;
- }
- }
-
- /**
- * Calculates the second moment.
- */
- public static class SecondMoment extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getSecondMoment() : Double.NaN;
- }
- }
-
- /**
- * Calculates the quadratic mean.
- */
- public static class QuadraticMean extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getQuadraticMean() : Double.NaN;
- }
- }
-
- /**
- * Calculates the standard deviation.
- */
- public static class StandardDeviation extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getStandardDeviation() : Double.NaN;
- }
- }
-
- /**
- * Calculates the sum of logs.
- */
- public static class SumLogs extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getSumOfLogs() : Double.NaN;
- }
- }
-
- /**
- * Calculates the sum of squares.
- */
- public static class SumSquares extends BaseStellarFunction {
- @Override
- public Object apply(List<Object> args) {
- SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
- return (stats != null) ? stats.getSumsq() : Double.NaN;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/86ce763e/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticalFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticalFunctionsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticalFunctionsTest.java
deleted file mode 100644
index 8019fb0..0000000
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticalFunctionsTest.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.metron.common.stellar;
-
-import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.StellarFunctions;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import static java.lang.String.format;
-
-/**
- * Tests the statistical summary functions of Stellar.
- */
-public class StellarStatisticalFunctionsTest {
-
- private List<Double> values;
- private Map<String, Object> variables;
- private SummaryStatistics stats;
-
- private static Object run(String expr, Map<String, Object> variables) {
- StellarProcessor processor = new StellarProcessor();
- assertTrue(processor.validate(expr));
- return processor.parse(expr, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT());
- }
-
- @Before
- public void setup() {
- variables = new HashMap<>();
-
- // initialize the statistical summary state
- Object result = run("STATS_INIT(0.0)", variables);
- assertNotNull(result);
- variables.put("stats", result);
-
- // add some values
- values = Arrays.asList(10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0);
- values.stream().forEach(val -> run(format("STATS_ADD (stats, %f)", val), variables));
-
- // add the same values to the StatisticalSummary object that is used for validation
- stats = new SummaryStatistics();
- values.stream().forEach(val -> stats.addValue(val));
- }
-
- @Test
- public void testAddManyIntegers() throws Exception {
- Object result = run("STATS_COUNT(stats)", variables);
- double countAtStart = (double) result;
-
- run("STATS_ADD(stats, 10, 20, 30, 40, 50)", variables);
-
- Object actual = run("STATS_COUNT(stats)", variables);
- assertEquals(countAtStart + 5.0, (double) actual, 0.1);
- }
-
- @Test
- public void testAddManyFloats() throws Exception {
- Object result = run("STATS_COUNT(stats)", variables);
- double countAtStart = (double) result;
-
- run("STATS_ADD(stats, 10.0, 20.0, 30.0, 40.0, 50.0)", variables);
-
- Object actual = run("STATS_COUNT(stats)", variables);
- assertEquals(countAtStart + 5.0, (double) actual, 0.1);
- }
-
- @Test
- public void testCount() throws Exception {
- Object actual = run("STATS_COUNT(stats)", variables);
- assertEquals(stats.getN(), (Double) actual, 0.1);
- }
-
- @Test
- public void testMean() throws Exception {
- Object actual = run("STATS_MEAN(stats)", variables);
- assertEquals(stats.getMean(), (Double) actual, 0.1);
- }
-
- @Test
- public void testGeometricMean() throws Exception {
- Object actual = run("STATS_GEOMETRIC_MEAN(stats)", variables);
- assertEquals(stats.getGeometricMean(), (Double) actual, 0.1);
- }
-
- @Test
- public void testMax() throws Exception {
- Object actual = run("STATS_MAX(stats)", variables);
- assertEquals(stats.getMax(), (Double) actual, 0.1);
- }
-
- @Test
- public void testMin() throws Exception {
- Object actual = run("STATS_MIN(stats)", variables);
- assertEquals(stats.getMin(), (Double) actual, 0.1);
- }
-
- @Test
- public void testSum() throws Exception {
- Object actual = run("STATS_SUM(stats)", variables);
- assertEquals(stats.getSum(), (Double) actual, 0.1);
- }
-
- @Test
- public void testPopulationVariance() throws Exception {
- Object actual = run("STATS_POPULATION_VARIANCE(stats)", variables);
- assertEquals(stats.getPopulationVariance(), (Double) actual, 0.1);
- }
-
- @Test
- public void testQuadraticMean() throws Exception {
- Object actual = run("STATS_QUADRATIC_MEAN(stats)", variables);
- assertEquals(stats.getQuadraticMean(), (Double) actual, 0.1);
- }
-
- @Test
- public void testSecondMoment() throws Exception {
- Object actual = run("STATS_SECOND_MOMENT(stats)", variables);
- assertEquals(stats.getSecondMoment(), (Double) actual, 0.1);
- }
-
- @Test
- public void testSumLogs() throws Exception {
- Object actual = run("STATS_SUM_LOGS(stats)", variables);
- assertEquals(stats.getSumOfLogs(), (Double) actual, 0.1);
- }
-
- @Test
- public void testStandardDeviation() throws Exception {
- Object actual = run("STATS_SD(stats)", variables);
- assertEquals(stats.getStandardDeviation(), (Double) actual, 0.1);
- }
-
- @Test
- public void testSumSquares() throws Exception {
- Object actual = run("STATS_SUM_SQUARES(stats)", variables);
- assertEquals(stats.getSumsq(), (Double) actual, 0.1);
- }
-
- @Test
- public void testVariance() throws Exception {
- Object actual = run("STATS_VARIANCE(stats)", variables);
- assertEquals(stats.getVariance(), (Double) actual, 0.1);
- }
-
- @Test
- public void testWithNull() throws Exception {
- Object actual = run("STATS_MEAN(null)", variables);
- assertTrue(((Double)actual).isNaN());
-
- actual = run("STATS_COUNT(null)", variables);
- assertTrue(((Double)actual).isNaN());
-
- actual = run("STATS_VARIANCE(null)", variables);
- assertTrue(((Double)actual).isNaN());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/86ce763e/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticsFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticsFunctionsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticsFunctionsTest.java
new file mode 100644
index 0000000..8c7ff70
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticsFunctionsTest.java
@@ -0,0 +1,270 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.common.stellar;
+
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.StellarFunctions;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static java.lang.String.format;
+
+/**
+ * Tests the statistical summary functions of Stellar.
+ */
+@RunWith(Parameterized.class)
+public class StellarStatisticsFunctionsTest {
+
+ private List<Double> values;
+ private Map<String, Object> variables;
+ private DescriptiveStatistics stats;
+ private SummaryStatistics summaryStats;
+ private int windowSize;
+
+ public StellarStatisticsFunctionsTest(int windowSize) {
+ this.windowSize = windowSize;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ // each test will be run against these values for windowSize
+ return Arrays.asList(new Object[][] {{ 0 }, { 100 }});
+ }
+
+ /**
+ * Runs a Stellar expression.
+ * @param expr The expression to run.
+ * @param variables The variables available to the expression.
+ */
+ private static Object run(String expr, Map<String, Object> variables) {
+ StellarProcessor processor = new StellarProcessor();
+ return processor.parse(expr, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT());
+ }
+
+ @Before
+ public void setup() {
+ variables = new HashMap<>();
+
+ // test input data
+ values = Arrays.asList(10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0);
+
+ // the DescriptiveStatistics is used for validation
+ stats = new DescriptiveStatistics(1000);
+ values.stream().forEach(val -> stats.addValue(val));
+
+ // the StatisticalSummary is used for validation
+ summaryStats = new SummaryStatistics();
+ values.stream().forEach(val -> summaryStats.addValue(val));
+ }
+
+ private void statsInit(int windowSize) {
+
+ // initialize
+ Object result = run("STATS_INIT(" + windowSize + ")", variables);
+ assertNotNull(result);
+ variables.put("stats", result);
+
+ // add some values
+ values = Arrays.asList(10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0);
+ values.stream().forEach(val -> run(format("STATS_ADD (stats, %f)", val), variables));
+ }
+
+ @Test
+ public void testAddManyIntegers() throws Exception {
+ statsInit(windowSize);
+ Object result = run("STATS_COUNT(stats)", variables);
+ double countAtStart = (double) result;
+
+ run("STATS_ADD(stats, 10, 20, 30, 40, 50)", variables);
+
+ Object actual = run("STATS_COUNT(stats)", variables);
+ assertEquals(countAtStart + 5.0, (double) actual, 0.1);
+ }
+
+ @Test
+ public void testAddManyFloats() throws Exception {
+ statsInit(windowSize);
+ Object result = run("STATS_COUNT(stats)", variables);
+ double countAtStart = (double) result;
+
+ run("STATS_ADD(stats, 10.0, 20.0, 30.0, 40.0, 50.0)", variables);
+
+ Object actual = run("STATS_COUNT(stats)", variables);
+ assertEquals(countAtStart + 5.0, (double) actual, 0.1);
+ }
+
+ @Test
+ public void testCount() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_COUNT(stats)", variables);
+ assertEquals(stats.getN(), (double) actual, 0.1);
+ }
+
+ @Test
+ public void testMean() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_MEAN(stats)", variables);
+ assertEquals(stats.getMean(), (Double) actual, 0.1);
+ }
+
+ @Test
+ public void testGeometricMean() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_GEOMETRIC_MEAN(stats)", variables);
+ assertEquals(stats.getGeometricMean(), (Double) actual, 0.1);
+ }
+
+ @Test
+ public void testMax() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_MAX(stats)", variables);
+ assertEquals(stats.getMax(), (Double) actual, 0.1);
+ }
+
+ @Test
+ public void testMin() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_MIN(stats)", variables);
+ assertEquals(stats.getMin(), (Double) actual, 0.1);
+ }
+
+ @Test
+ public void testSum() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_SUM(stats)", variables);
+ assertEquals(stats.getSum(), (Double) actual, 0.1);
+ }
+
+ @Test
+ public void testStandardDeviation() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_SD(stats)", variables);
+ assertEquals(stats.getStandardDeviation(), (Double) actual, 0.1);
+ }
+
+ @Test
+ public void testVariance() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_VARIANCE(stats)", variables);
+ assertEquals(stats.getVariance(), (Double) actual, 0.1);
+ }
+
+ @Test
+ public void testPopulationVariance() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_POPULATION_VARIANCE(stats)", variables);
+ assertEquals(stats.getPopulationVariance(), (Double) actual, 0.1);
+ }
+
+ @Test
+ public void testQuadraticMean() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_QUADRATIC_MEAN(stats)", variables);
+ assertEquals(stats.getQuadraticMean(), (Double) actual, 0.1);
+ }
+
+ @Test
+ public void testSumLogsNoWindow() throws Exception {
+ statsInit(0);
+ Object actual = run("STATS_SUM_LOGS(stats)", variables);
+ assertEquals(summaryStats.getSumOfLogs(), (Double) actual, 0.1);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testSumLogsWithWindow() throws Exception {
+ statsInit(100);
+ run("STATS_SUM_LOGS(stats)", variables);
+ }
+
+ @Test
+ public void testSumSquares() throws Exception {
+ statsInit(windowSize);
+ Object actual = run("STATS_SUM_SQUARES(stats)", variables);
+ assertEquals(stats.getSumsq(), (Double) actual, 0.1);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testKurtosisNoWindow() throws Exception {
+ statsInit(0);
+ run("STATS_KURTOSIS(stats)", variables);
+ }
+
+ @Test
+ public void testKurtosisWithWindow() throws Exception {
+ statsInit(100);
+ Object actual = run("STATS_KURTOSIS(stats)", variables);
+ assertEquals(stats.getKurtosis(), (Double) actual, 0.1);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testSkewnessNoWindow() throws Exception {
+ statsInit(0);
+ run("STATS_SKEWNESS(stats)", variables);
+ }
+
+ @Test
+ public void testSkewnessWithWindow() throws Exception {
+ statsInit(100);
+ Object actual = run("STATS_SKEWNESS(stats)", variables);
+ assertEquals(stats.getSkewness(), (Double) actual, 0.1);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testPercentileNoWindow() throws Exception {
+ statsInit(0);
+ final double percentile = 0.9;
+ Object actual = run(format("STATS_PERCENTILE(stats, %f)", percentile), variables);
+ }
+
+ @Test
+ public void testPercentileWithWindow() throws Exception {
+ statsInit(100);
+ final double percentile = 0.9;
+ Object actual = run(format("STATS_PERCENTILE(stats, %f)", percentile), variables);
+ assertEquals(stats.getPercentile(percentile), (Double) actual, 0.1);
+ }
+
+ @Test
+ public void testWithNull() throws Exception {
+ Object actual = run("STATS_MEAN(null)", variables);
+ assertTrue(((Double)actual).isNaN());
+
+ actual = run("STATS_COUNT(null)", variables);
+ assertTrue(((Double)actual).isNaN());
+
+ actual = run("STATS_VARIANCE(null)", variables);
+ assertTrue(((Double)actual).isNaN());
+ }
+}
\ No newline at end of file