[2/3] incubator-metron git commit: METRON-562: Add rudimentary statistical outlier detection closes apache/incubator-metron#352

[ce...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java
new file mode 100644
index 0000000..b9846e7
--- /dev/null
+++ b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java
@@ -0,0 +1,387 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.metron.statistics;
+
+import com.google.common.base.Joiner;
+import org.apache.commons.math3.random.GaussianRandomGenerator;
+import org.apache.commons.math3.random.MersenneTwister;
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.StellarFunctions;
+import org.apache.metron.common.stellar.StellarProcessor;
+import org.apache.metron.common.utils.SerDeUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.*;
+import java.util.function.Function;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static java.lang.String.format;
+
+/**
+ * Tests the statistical summary functions of Stellar.
+ */
+@RunWith(Parameterized.class)
+public class StellarStatisticsFunctionsTest {
+
+  private List<Double> values;
+  private Map<String, Object> variables;
+  private DescriptiveStatistics stats;
+  private SummaryStatistics summaryStats;
+  private int windowSize;
+
+  public StellarStatisticsFunctionsTest(int windowSize) {
+    this.windowSize = windowSize;
+  }
+
+  @Parameterized.Parameters
+  public static Collection<Object[]> data() {
+    // each test will be run against these values for windowSize
+    return Arrays.asList(new Object[][] {{ 0 }, { 100 }});
+  }
+
+  private static void tolerantAssertEquals( Function<StatisticsProvider, Number> func
+                                          , StatisticsProvider left
+                                          , StatisticsProvider right
+                                          )
+
+  {
+    tolerantAssertEquals(func, left, right, null);
+  }
+
+  private static void tolerantAssertEquals( Function<StatisticsProvider, Number> func
+                                          , StatisticsProvider left
+                                          , StatisticsProvider right
+                                          , Double epsilon
+                                          )
+  {
+    try {
+      Number leftVal = func.apply(left);
+      Number rightVal = func.apply(left);
+      if(epsilon != null) {
+        Assert.assertEquals((double)leftVal, (double)rightVal, epsilon);
+      }
+      else {
+        Assert.assertEquals(leftVal, rightVal);
+      }
+    }
+    catch(UnsupportedOperationException uoe) {
+      //ignore
+    }
+
+  }
+
+  /**
+   * Runs a Stellar expression.
+   * @param expr The expression to run.
+   * @param variables The variables available to the expression.
+   */
+  private static Object run(String expr, Map<String, Object> variables) {
+    StellarProcessor processor = new StellarProcessor();
+    Object ret = processor.parse(expr, x-> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT());
+    byte[] raw = SerDeUtils.toBytes(ret);
+    Object actual = SerDeUtils.fromBytes(raw, Object.class);
+    if(ret instanceof StatisticsProvider) {
+      StatisticsProvider left = (StatisticsProvider)ret;
+      StatisticsProvider right = (StatisticsProvider)actual;
+      //N
+      tolerantAssertEquals(prov -> prov.getCount(), left, right);
+      //sum
+      tolerantAssertEquals(prov -> prov.getSum(), left, right, 1e-3);
+      //sum of squares
+      tolerantAssertEquals(prov -> prov.getSumSquares(), left, right, 1e-3);
+      //sum of squares
+      tolerantAssertEquals(prov -> prov.getSumLogs(), left, right, 1e-3);
+      //Mean
+      tolerantAssertEquals(prov -> prov.getMean(), left, right, 1e-3);
+      //Quadratic Mean
+      tolerantAssertEquals(prov -> prov.getQuadraticMean(), left, right, 1e-3);
+      //SD
+      tolerantAssertEquals(prov -> prov.getStandardDeviation(), left, right, 1e-3);
+      //Variance
+      tolerantAssertEquals(prov -> prov.getVariance(), left, right, 1e-3);
+      //Min
+      tolerantAssertEquals(prov -> prov.getMin(), left, right, 1e-3);
+      //Max
+      tolerantAssertEquals(prov -> prov.getMax(), left, right, 1e-3);
+      //Kurtosis
+      tolerantAssertEquals(prov -> prov.getKurtosis(), left, right, 1e-3);
+      //Skewness
+      tolerantAssertEquals(prov -> prov.getSkewness(), left, right, 1e-3);
+      for (double d = 10.0; d < 100.0; d += 10) {
+        final double pctile = d;
+        //This is a sketch, so we're a bit more forgiving here in our choice of \epsilon.
+        tolerantAssertEquals(prov -> prov.getPercentile(pctile), left, right, 1e-2);
+
+      }
+    }
+    return ret;
+  }
+
+  @Before
+  public void setup() {
+    variables = new HashMap<>();
+
+    // test input data
+    values = Arrays.asList(10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0);
+
+    // the DescriptiveStatistics is used for validation
+    stats = new DescriptiveStatistics(1000);
+    values.stream().forEach(val -> stats.addValue(val));
+
+    // the StatisticalSummary is used for validation
+    summaryStats = new SummaryStatistics();
+    values.stream().forEach(val -> summaryStats.addValue(val));
+  }
+
+  private void statsInit(int windowSize) {
+
+    // initialize
+    Object result = run("STATS_INIT(" + windowSize + ")", variables);
+    assertNotNull(result);
+    variables.put("stats", result);
+
+    // add some values
+    values.stream().forEach(val -> run(format("STATS_ADD (stats, %f)", val), variables));
+  }
+
+  @Test(expected=ParseException.class)
+  public void testOverflow() throws Exception {
+   run(format("STATS_ADD(STATS_INIT(), %f)", (Double.MAX_VALUE + 1)), new HashMap<>());
+  }
+
+  @Test
+  public void ensureDeterminism() throws Exception {
+    for(int i = 0;i < 20;++i) {
+      testMergeProviders();
+    }
+  }
+
+  @Test
+  public void testMergeProviders() throws Exception {
+    List<StatisticsProvider> providers = new ArrayList<>();
+    /*
+    Create 10 providers, each with a sample drawn from a gaussian distribution.
+    Update the reference stats from commons math to ensure we are
+     */
+    GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(1L));
+    SummaryStatistics sStatistics= new SummaryStatistics();
+    DescriptiveStatistics dStatistics = new DescriptiveStatistics();
+    for(int i = 0;i < 10;++i) {
+      List<Double> sample = new ArrayList<>();
+      for(int j = 0;j < 100;++j) {
+        double s = gaussian.nextNormalizedDouble();
+        sample.add(s);
+        sStatistics.addValue(s);
+        dStatistics.addValue(s);
+      }
+      StatisticsProvider provider = (StatisticsProvider)run("STATS_ADD(STATS_INIT(), " + Joiner.on(",").join(sample) + ")"
+                                                           , new HashMap<>()
+                                                           );
+      providers.add(provider);
+    }
+
+    /*
+    Merge the providers and validate
+     */
+    Map<String, Object> providerVariables = new HashMap<>();
+    for(int i = 0;i < providers.size();++i) {
+      providerVariables.put("provider_" + i, providers.get(i));
+    }
+    StatisticsProvider mergedProvider =
+            (StatisticsProvider)run("STATS_MERGE([" + Joiner.on(",").join(providerVariables.keySet()) + "])"
+                                   , providerVariables
+                                   );
+    OnlineStatisticsProviderTest.validateStatisticsProvider(mergedProvider, sStatistics , dStatistics);
+
+  }
+
+  @Test
+  public void testAddManyIntegers() throws Exception {
+    statsInit(windowSize);
+    Object result = run("STATS_COUNT(stats)", variables);
+    double countAtStart = (double) result;
+
+    run("STATS_ADD(stats, 10, 20, 30, 40, 50)", variables);
+
+    Object actual = run("STATS_COUNT(stats)", variables);
+    assertEquals(countAtStart + 5.0, (double) actual, 0.1);
+  }
+
+  @Test
+  public void testAddManyFloats() throws Exception {
+    statsInit(windowSize);
+    Object result = run("STATS_COUNT(stats)", variables);
+    double countAtStart = (double) result;
+
+    run("STATS_ADD(stats, 10.0, 20.0, 30.0, 40.0, 50.0)", variables);
+
+    Object actual = run("STATS_COUNT(stats)", variables);
+    assertEquals(countAtStart + 5.0, (double) actual, 0.1);
+  }
+
+  @Test
+  public void testCount() throws Exception {
+    statsInit(windowSize);
+    Object actual = run("STATS_COUNT(stats)", variables);
+    assertEquals(stats.getN(), (double) actual, 0.1);
+  }
+
+  @Test
+  public void testMean() throws Exception {
+    statsInit(windowSize);
+    Object actual = run("STATS_MEAN(stats)", variables);
+    assertEquals(stats.getMean(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testGeometricMean() throws Exception {
+    if(windowSize > 0) {
+      statsInit(windowSize);
+      Object actual = run("STATS_GEOMETRIC_MEAN(stats)", variables);
+      assertEquals(stats.getGeometricMean(), (Double) actual, 0.1);
+    }
+  }
+
+  @Test
+  public void testMax() throws Exception {
+    statsInit(windowSize);
+    Object actual = run("STATS_MAX(stats)", variables);
+    assertEquals(stats.getMax(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testMin() throws Exception {
+    statsInit(windowSize);
+    Object actual = run("STATS_MIN(stats)", variables);
+    assertEquals(stats.getMin(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testSum() throws Exception {
+    statsInit(windowSize);
+    Object actual = run("STATS_SUM(stats)", variables);
+    assertEquals(stats.getSum(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testStandardDeviation() throws Exception {
+    statsInit(windowSize);
+    Object actual = run("STATS_SD(stats)", variables);
+    assertEquals(stats.getStandardDeviation(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testVariance() throws Exception {
+    statsInit(windowSize);
+    Object actual = run("STATS_VARIANCE(stats)", variables);
+    assertEquals(stats.getVariance(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testPopulationVariance() throws Exception {
+    if(windowSize > 0) {
+      statsInit(windowSize);
+      Object actual = run("STATS_POPULATION_VARIANCE(stats)", variables);
+      assertEquals(stats.getPopulationVariance(), (Double) actual, 0.1);
+    }
+  }
+
+  @Test
+  public void testQuadraticMean() throws Exception {
+    if(windowSize > 0) {
+      statsInit(windowSize);
+      Object actual = run("STATS_QUADRATIC_MEAN(stats)", variables);
+      assertEquals(stats.getQuadraticMean(), (Double) actual, 0.1);
+    }
+  }
+
+  @Test
+  public void testSumLogsNoWindow() throws Exception {
+    statsInit(0);
+    Object actual = run("STATS_SUM_LOGS(stats)", variables);
+    assertEquals(summaryStats.getSumOfLogs(), (Double) actual, 0.1);
+  }
+
+  @Test(expected = ParseException.class)
+  public void testSumLogsWithWindow() throws Exception {
+    statsInit(100);
+    run("STATS_SUM_LOGS(stats)", variables);
+  }
+
+  @Test
+  public void testSumSquares() throws Exception {
+    statsInit(windowSize);
+    Object actual = run("STATS_SUM_SQUARES(stats)", variables);
+    assertEquals(stats.getSumsq(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testKurtosis() throws Exception {
+    statsInit(windowSize);
+    Object actual = run("STATS_KURTOSIS(stats)", variables);
+    assertEquals(stats.getKurtosis(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testSkewness() throws Exception {
+    statsInit(windowSize);
+    Object actual = run("STATS_SKEWNESS(stats)", variables);
+    assertEquals(stats.getSkewness(), (Double) actual, 0.1);
+  }
+
+
+  @Test
+  public void testPercentileNoWindow() throws Exception {
+    statsInit(0);
+    final double percentile = 0.9;
+    Object actual = run(format("STATS_PERCENTILE(stats, %f)", percentile), variables);
+    assertEquals(stats.getPercentile(percentile), (Double) actual, 1);
+  }
+
+  @Test
+  public void testPercentileWithWindow() throws Exception {
+    statsInit(100);
+    final double percentile = 0.9;
+    Object actual = run(format("STATS_PERCENTILE(stats, %f)", percentile), variables);
+    assertEquals(stats.getPercentile(percentile), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testWithNull() throws Exception {
+    Object actual = run("STATS_MEAN(null)", variables);
+    assertTrue(((Double)actual).isNaN());
+
+    actual = run("STATS_COUNT(null)", variables);
+    assertTrue(((Double)actual).isNaN());
+
+    actual = run("STATS_VARIANCE(null)", variables);
+    assertTrue(((Double)actual).isNaN());
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationTest.java b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationTest.java
new file mode 100644
index 0000000..952da8d
--- /dev/null
+++ b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationTest.java
@@ -0,0 +1,159 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.statistics.outlier;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.math3.distribution.TDistribution;
+import org.apache.commons.math3.random.GaussianRandomGenerator;
+import org.apache.commons.math3.random.MersenneTwister;
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.StellarFunctions;
+import org.apache.metron.common.stellar.StellarProcessor;
+import org.apache.metron.common.utils.SerDeUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class MedianAbsoluteDeviationTest {
+  public static Object run(String rule, Map<String, Object> variables) {
+    Context context = Context.EMPTY_CONTEXT();
+    StellarProcessor processor = new StellarProcessor();
+    Assert.assertTrue(rule + " not valid.", processor.validate(rule, context));
+    return processor.parse(rule, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), context);
+  }
+
+  private void assertScoreEquals(MedianAbsoluteDeviationFunctions.State currentState, MedianAbsoluteDeviationFunctions.State clonedState, double value) {
+     Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", value));
+      Double clonedScore = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", clonedState, "value", value));
+      Assert.assertEquals(score, clonedScore, 1e-6);
+  }
+
+  @Test
+  public void testSerialization() {
+    MedianAbsoluteDeviationFunctions.State currentState = null;
+    List<MedianAbsoluteDeviationFunctions.State> states = new ArrayList<>();
+    currentState = (MedianAbsoluteDeviationFunctions.State) run("OUTLIER_MAD_STATE_MERGE(states, NULL)", ImmutableMap.of("states", states));
+    for(int i = 0;i < 100;++i) {
+      double d = 1.2*i;
+      run("OUTLIER_MAD_ADD(currentState, data)", ImmutableMap.of("currentState", currentState, "data", d));
+    }
+
+    byte[] stateBytes = SerDeUtils.toBytes(currentState);
+    MedianAbsoluteDeviationFunctions.State clonedState = SerDeUtils.fromBytes(stateBytes, MedianAbsoluteDeviationFunctions.State.class);
+    assertScoreEquals(currentState, clonedState, 0d);
+    assertScoreEquals(currentState, clonedState, 1d);
+    assertScoreEquals(currentState, clonedState, 10d);
+  }
+
+
+  @Test
+  public void test() {
+
+    GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(0L));
+    DescriptiveStatistics stats = new DescriptiveStatistics();
+    List<MedianAbsoluteDeviationFunctions.State> states = new ArrayList<>();
+    MedianAbsoluteDeviationFunctions.State currentState = null;
+    //initialize the state
+    currentState = (MedianAbsoluteDeviationFunctions.State) run("OUTLIER_MAD_STATE_MERGE(states, NULL)", ImmutableMap.of("states", states));
+    for(int i = 0,j=0;i < 10000;++i,++j) {
+      Double d = gaussian.nextNormalizedDouble();
+      stats.addValue(d);
+      run("OUTLIER_MAD_ADD(currentState, data)", ImmutableMap.of("currentState", currentState, "data", d));
+      if(j >= 1000) {
+        j = 0;
+        List<MedianAbsoluteDeviationFunctions.State> stateWindow = new ArrayList<>();
+        for(int stateIndex = Math.max(0, states.size() - 5);stateIndex < states.size();++stateIndex) {
+          stateWindow.add(states.get(stateIndex));
+        }
+        currentState = (MedianAbsoluteDeviationFunctions.State) run("OUTLIER_MAD_STATE_MERGE(states, currentState)"
+                , ImmutableMap.of("states", stateWindow, "currentState", currentState));
+      }
+    }
+    {
+      Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMin()));
+      Assert.assertTrue("Score: " + score + " is not an outlier despite being a minimum.", score > 3.5);
+    }
+    {
+      Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMax()));
+      Assert.assertTrue("Score: " + score + " is not an outlier despite being a maximum", score > 3.5);
+    }
+    {
+      Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean() + 4*stats.getStandardDeviation()));
+      Assert.assertTrue("Score: " + score + " is not an outlier despite being 4 std deviations away from the mean", score > 3.5);
+    }
+    {
+      Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean() - 4*stats.getStandardDeviation()));
+      Assert.assertTrue("Score: " + score + " is not an outlier despite being 4 std deviations away from the mean", score > 3.5);
+    }
+    {
+      Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean()));
+      Assert.assertFalse("Score: " + score + " is an outlier despite being the mean", score > 3.5);
+    }
+  }
+
+  @Test
+  public void testLongTailed() {
+
+    TDistribution generator = new TDistribution(new MersenneTwister(0L), 100);
+    DescriptiveStatistics stats = new DescriptiveStatistics();
+    List<MedianAbsoluteDeviationFunctions.State> states = new ArrayList<>();
+    MedianAbsoluteDeviationFunctions.State currentState = null;
+    //initialize the state
+    currentState = (MedianAbsoluteDeviationFunctions.State) run("OUTLIER_MAD_STATE_MERGE(states, NULL)", ImmutableMap.of("states", states));
+    for(int i = 0,j=0;i < 10000;++i,++j) {
+      Double d = generator.sample();
+      stats.addValue(d);
+      run("OUTLIER_MAD_ADD(currentState, data)", ImmutableMap.of("currentState", currentState, "data", d));
+      if(j >= 1000) {
+        j = 0;
+        List<MedianAbsoluteDeviationFunctions.State> stateWindow = new ArrayList<>();
+        for(int stateIndex = Math.max(0, states.size() - 5);stateIndex < states.size();++stateIndex) {
+          stateWindow.add(states.get(stateIndex));
+        }
+        currentState = (MedianAbsoluteDeviationFunctions.State) run("OUTLIER_MAD_STATE_MERGE(states, currentState)"
+                , ImmutableMap.of("states", stateWindow, "currentState", currentState));
+      }
+    }
+    {
+      Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMin()));
+      Assert.assertTrue("Score: " + score + " is not an outlier despite being a minimum.", score > 3.5);
+    }
+    {
+      Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMax()));
+      Assert.assertTrue("Score: " + score + " is not an outlier despite being a maximum", score > 3.5);
+    }
+    {
+      Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean() + 4*stats.getStandardDeviation()));
+      Assert.assertTrue("Score: " + score + " is not an outlier despite being 4 std deviations away from the mean", score > 3.5);
+    }
+    {
+      Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean() - 4*stats.getStandardDeviation()));
+      Assert.assertTrue("Score: " + score + " is not an outlier despite being 4 std deviations away from the mean", score > 3.5);
+    }
+    {
+      Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean()));
+      Assert.assertFalse("Score: " + score + " is an outlier despite being the mean", score > 3.5);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-analytics/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/pom.xml b/metron-analytics/pom.xml
index 07d192c..6e7e238 100644
--- a/metron-analytics/pom.xml
+++ b/metron-analytics/pom.xml
@@ -42,6 +42,7 @@
 	<modules>
 		<module>metron-maas-service</module>
 		<module>metron-maas-common</module>
+		<module>metron-statistics</module>
 		<module>metron-profiler</module>
 		<module>metron-profiler-client</module>
 		<module>metron-profiler-common</module>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-platform/metron-common/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md
index 1444345..25bacb7 100644
--- a/metron-platform/metron-common/README.md
+++ b/metron-platform/metron-common/README.md
@@ -392,116 +392,6 @@ MAP_GET`
     * prefix - The proposed prefix
   * Returns: True if the string starts with the specified prefix and false if otherwise
 
-### `STATS_ADD`
-  * Description: Adds one or more input values to those that are used to calculate the summary statistics.
-  * Input:
-    * stats - The Stellar statistics object.  If null, then a new one is initialized.
-    * value+ - One or more numbers to add
-  * Returns: A Stellar statistics object
-
-### `STATS_COUNT`
-  * Description: Calculates the count of the values accumulated (or in the window if a window is used).
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The count of the values in the window or NaN if the statistics object is null.
-
-### `STATS_GEOMETRIC_MEAN`
-  * Description: Calculates the geometric mean of the accumulated values (or in the window if a window is used). See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics 
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The geometric mean of the values in the window or NaN if the statistics object is null.
-
-### `STATS_INIT`
-  * Description: Initializes a statistics object
-  * Input:
-    * window_size - The number of input data values to maintain in a rolling window in memory.  If window_size is equal to 0, then no rolling window is maintained. Using no rolling window is less memory intensive, but cannot calculate certain statistics like percentiles and kurtosis.
-  * Returns: A Stellar statistics object
-
-### `STATS_KURTOSIS`
-  * Description: Calculates the kurtosis of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics 
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The kurtosis of the values in the window or NaN if the statistics object is null.
-
-### `STATS_MAX`
-  * Description: Calculates the maximum of the accumulated values (or in the window if a window is used).
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The maximum of the accumulated values in the window or NaN if the statistics object is null.
-
-### `STATS_MEAN`
-  * Description: Calculates the mean of the accumulated values (or in the window if a window is used).
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The mean of the values in the window or NaN if the statistics object is null.
-
-### `STATS_MERGE`
-  * Description: Merges statistics objects.
-  * Input:
-    * statistics - A list of statistics objects
-  * Returns: A Stellar statistics object
-
-### `STATS_MIN`
-  * Description: Calculates the minimum of the accumulated values (or in the window if a window is used).
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The minimum of the accumulated values in the window or NaN if the statistics object is null.
-
-### `STATS_PERCENTILE`
-  * Description: Computes the p'th percentile of the accumulated values (or in the window if a window is used).
-  * Input:
-    * stats - The Stellar statistics object
-    * p - a double where 0 <= p < 1 representing the percentile
-  * Returns: The p'th percentile of the data or NaN if the statistics object is null
-
-### `STATS_POPULATION_VARIANCE`
-  * Description: Calculates the population variance of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics 
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The population variance of the values in the window or NaN if the statistics object is null.
-
-### `STATS_QUADRATIC_MEAN`
-  * Description: Calculates the quadratic mean of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics 
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The quadratic mean of the values in the window or NaN if the statistics object is null.
-
-### `STATS_SD`
-  * Description: Calculates the standard deviation of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics 
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The standard deviation of the values in the window or NaN if the statistics object is null.
-
-### `STATS_SKEWNESS`
-  * Description: Calculates the skewness of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics 
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The skewness of the values in the window or NaN if the statistics object is null.
-
-### `STATS_SUM`
-  * Description: Calculates the sum of the accumulated values (or in the window if a window is used).
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The sum of the values in the window or NaN if the statistics object is null.
-
-### `STATS_SUM_LOGS`
-  * Description: Calculates the sum of the (natural) log of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics 
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The sum of the (natural) log of the values in the window or NaN if the statistics object is null.
-
-### `STATS_SUM_SQUARES`
-  * Description: Calculates the sum of the squares of the accumulated values (or in the window if a window is used).
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The sum of the squares of the values in the window or NaN if the statistics object is null.
-
-### `STATS_VARIANCE`
-  * Description: Calculates the variance of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics 
-  * Input:
-    * stats - The Stellar statistics object
-  * Returns: The variance of the values in the window or NaN if the statistics object is null.
-
 ### `SYSTEM_ENV_GET`
   * Description: Returns the value associated with an environment variable
   * Input:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index c05111f..60c5b26 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -317,11 +317,6 @@
             <version>0.38</version>
         </dependency>
         <dependency>
-            <groupId>com.tdunning</groupId>
-            <artifactId>t-digest</artifactId>
-            <version>3.1</version>
-        </dependency>
-        <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-collections4</artifactId>
             <version>4.1</version>
@@ -427,14 +422,7 @@
                                     <pattern>org.apache.commons.beanutils</pattern>
                                     <shadedPattern>org.apache.metron.beanutils</shadedPattern>
                                 </relocation>
-                                <relocation>
-                                    <pattern>com.esotericsoftware</pattern>
-                                    <shadedPattern>org.apache.metron.esotericsoftware</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>de.javakaffee</pattern>
-                                    <shadedPattern>org.apache.metron.javakaffee</shadedPattern>
-                                </relocation>
+
                             </relocations>
                             <transformers>
                                 <transformer

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
index 4cf3ac2..d6f1a37 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
@@ -18,10 +18,7 @@
 package org.apache.metron.common.configuration.profiler;
 
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -67,6 +64,13 @@ public class ProfileConfig implements Serializable {
    */
   private Map<String, String> update = new HashMap<>();
 
+   /**
+   * A set of expressions that is executed when a tick happens.
+   * A map is expected where the key is the variable name and the value is a Stellar
+   * expression.  The map can include 0 or more variables/expressions.
+   */
+  private Map<String, String> tickUpdate = new LinkedHashMap<>();
+
   /**
    * A list of Stellar expressions that is executed in order and used to group the
    * resulting profile data.
@@ -127,6 +131,14 @@ public class ProfileConfig implements Serializable {
     this.update = update;
   }
 
+  public Map<String, String> getTickUpdate() {
+    return tickUpdate;
+  }
+
+  public void setTickUpdate(Map<String, String> tickUpdate) {
+    this.tickUpdate = tickUpdate;
+  }
+
   public List<String> getGroupBy() {
     return groupBy;
   }
@@ -152,6 +164,21 @@ public class ProfileConfig implements Serializable {
   }
 
   @Override
+  public String toString() {
+    return "ProfileConfig{" +
+            "profile='" + profile + '\'' +
+            ", foreach='" + foreach + '\'' +
+            ", onlyif='" + onlyif + '\'' +
+            ", init=" + init +
+            ", update=" + update +
+            ", tickUpdate=" + tickUpdate +
+            ", groupBy=" + groupBy +
+            ", result='" + result + '\'' +
+            ", expires=" + expires +
+            '}';
+  }
+
+  @Override
   public boolean equals(Object o) {
     if (this == o) return true;
     if (o == null || getClass() != o.getClass()) return false;
@@ -163,6 +190,7 @@ public class ProfileConfig implements Serializable {
     if (onlyif != null ? !onlyif.equals(that.onlyif) : that.onlyif != null) return false;
     if (init != null ? !init.equals(that.init) : that.init != null) return false;
     if (update != null ? !update.equals(that.update) : that.update != null) return false;
+    if (tickUpdate != null ? !tickUpdate.equals(that.tickUpdate) : that.tickUpdate != null) return false;
     if (groupBy != null ? !groupBy.equals(that.groupBy) : that.groupBy != null) return false;
     if (result != null ? !result.equals(that.result) : that.result != null) return false;
     return expires != null ? expires.equals(that.expires) : that.expires == null;
@@ -176,23 +204,10 @@ public class ProfileConfig implements Serializable {
     result1 = 31 * result1 + (onlyif != null ? onlyif.hashCode() : 0);
     result1 = 31 * result1 + (init != null ? init.hashCode() : 0);
     result1 = 31 * result1 + (update != null ? update.hashCode() : 0);
+    result1 = 31 * result1 + (tickUpdate != null ? tickUpdate.hashCode() : 0);
     result1 = 31 * result1 + (groupBy != null ? groupBy.hashCode() : 0);
     result1 = 31 * result1 + (result != null ? result.hashCode() : 0);
     result1 = 31 * result1 + (expires != null ? expires.hashCode() : 0);
     return result1;
   }
-
-  @Override
-  public String toString() {
-    return "ProfileConfig{" +
-            "profile='" + profile + '\'' +
-            ", foreach='" + foreach + '\'' +
-            ", onlyif='" + onlyif + '\'' +
-            ", init=" + init +
-            ", update=" + update +
-            ", groupBy=" + groupBy +
-            ", result='" + result + '\'' +
-            ", expires=" + expires +
-            '}';
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatisticsFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatisticsFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatisticsFunctions.java
deleted file mode 100644
index d37f9e3..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StellarStatisticsFunctions.java
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- *
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-
-package org.apache.metron.common.dsl.functions;
-
-import org.apache.metron.common.dsl.BaseStellarFunction;
-import org.apache.metron.common.dsl.Stellar;
-import org.apache.metron.common.math.stats.OnlineStatisticsProvider;
-import org.apache.metron.common.math.stats.StatisticsProvider;
-import org.apache.metron.common.math.stats.WindowedStatisticsProvider;
-
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.metron.common.utils.ConversionUtils.convert;
-
-/**
- * Provides Stellar functions that can calculate summary statistics on
- * streams of data.
- */
-public class StellarStatisticsFunctions {
-
-  /**
-   * Initializes the summary statistics.
-   *
-   * Initialization can occur from either STATS_INIT and STATS_ADD.
-   */
-  private static StatisticsProvider statsInit(List<Object> args) {
-    int windowSize = 0;
-    if(args.size() > 0 && args.get(0) instanceof Number) {
-      windowSize = convert(args.get(0), Integer.class);
-    }
-    if(windowSize > 0) {
-      return new WindowedStatisticsProvider(windowSize);
-    }
-    return new OnlineStatisticsProvider();
-  }
-
-  @Stellar( namespace="STATS"
-          , name="MERGE"
-          , description = "Merges statistics objects."
-          , params = {
-                      "statistics - A list of statistics objects"
-                      }
-          , returns = "A Stellar statistics object"
-          )
-  public static class Merge extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      if(args.size() > 0) {
-        Object firstArg = args.get(0);
-        if(firstArg instanceof List) {
-          StatisticsProvider ret = null;
-          for(Object sp : (List)firstArg) {
-            if(sp instanceof StatisticsProvider) {
-              if(ret == null) {
-                ret = (StatisticsProvider)sp;
-              }
-              else {
-                ret = ret.merge((StatisticsProvider)sp);
-              }
-            }
-          }
-          return ret;
-        }
-        else {
-          return null;
-        }
-      }
-      return null;
-    }
-  }
-
-  /**
-   * Initialize the summary statistics.
-   *
-   *  STATS_INIT (window_size)
-   *
-   * window_size The number of input data values to maintain in a rolling window
-   *             in memory.  If equal to 0, then no rolling window is maintained.
-   *             Using no rolling window is less memory intensive, but cannot
-   *             calculate certain statistics like percentiles and kurtosis.
-   */
-  @Stellar( namespace="STATS"
-          , name="INIT"
-          , description = "Initializes a statistics object"
-          , params = {
-                      "window_size - The number of input data values to maintain in a rolling window " +
-                      "in memory.  If window_size is equal to 0, then no rolling window is maintained. " +
-                      "Using no rolling window is less memory intensive, but cannot " +
-                      "calculate certain statistics like percentiles and kurtosis."
-                      }
-          , returns = "A Stellar statistics object"
-          )
-  public static class Init extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      return statsInit(args);
-    }
-  }
-
-  /**
-   * Add an input value to those that are used to calculate the summary statistics.
-   *
-   *  STATS_ADD (stats, value [, value2, value3, ...])
-   */
-  @Stellar(namespace="STATS"
-          , name="ADD"
-          , description = "Adds one or more input values to those that are used to calculate the summary statistics."
-          , params = {
-                      "stats - The Stellar statistics object.  If null, then a new one is initialized."
-                     , "value+ - One or more numbers to add"
-                     }
-          , returns = "A Stellar statistics object"
-          )
-  public static class Add extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-
-      // initialize a stats object, if one does not already exist
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      if(stats == null) {
-        stats = statsInit(Collections.emptyList());
-      }
-
-      // add each of the numeric values
-      for(int i=1; i<args.size(); i++) {
-        double value = convert(args.get(i), Double.class);
-        stats.addValue(value);
-      }
-
-      return stats;
-    }
-  }
-
-  /**
-   * Calculates the mean.
-   *
-   *  STATS_MEAN (stats)
-   */
-  @Stellar( namespace="STATS"
-          , name="MEAN"
-          , description = "Calculates the mean of the accumulated values (or in the window if a window is used)."
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The mean of the values in the window or NaN if the statistics object is null."
-          )
-  public static class Mean extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getMean() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the geometric mean.
-   */
-  @Stellar( namespace="STATS"
-          , name="GEOMETRIC_MEAN"
-          , description = "Calculates the geometric mean of the accumulated values (or in the window if a window is used). See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics "
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The geometric mean of the values in the window or NaN if the statistics object is null."
-          )
-  public static class GeometricMean extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getGeometricMean() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the sum.
-   */
-  @Stellar(namespace="STATS"
-          , name="SUM"
-          , description = "Calculates the sum of the accumulated values (or in the window if a window is used)."
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The sum of the values in the window or NaN if the statistics object is null."
-          )
-  public static class Sum extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getSum() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the max.
-   */
-  @Stellar(namespace="STATS", name="MAX"
-          , description = "Calculates the maximum of the accumulated values (or in the window if a window is used)."
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The maximum of the accumulated values in the window or NaN if the statistics object is null."
-          )
-  public static class Max extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getMax() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the min.
-   */
-  @Stellar(namespace="STATS", name="MIN"
-          , description = "Calculates the minimum of the accumulated values (or in the window if a window is used)."
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The minimum of the accumulated values in the window or NaN if the statistics object is null."
-          )
-  public static class Min extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getMin() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the count of elements
-   */
-  @Stellar(namespace="STATS", name="COUNT"
-          , description = "Calculates the count of the values accumulated (or in the window if a window is used)."
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The count of the values in the window or NaN if the statistics object is null.")
-  public static class Count extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? convert(stats.getCount(), Double.class) : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the population variance.
-   */
-  @Stellar(namespace="STATS", name="POPULATION_VARIANCE"
-          , description = "Calculates the population variance of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics "
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The population variance of the values in the window or NaN if the statistics object is null.")
-  public static class PopulationVariance extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getPopulationVariance() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the variance.
-   */
-  @Stellar(namespace="STATS", name="VARIANCE"
-          , description = "Calculates the variance of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics "
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The variance of the values in the window or NaN if the statistics object is null.")
-  public static class Variance extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getVariance() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the quadratic mean.
-   */
-  @Stellar(namespace="STATS", name="QUADRATIC_MEAN"
-          , description = "Calculates the quadratic mean of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics "
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The quadratic mean of the values in the window or NaN if the statistics object is null.")
-  public static class QuadraticMean extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getQuadraticMean() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the standard deviation.
-   */
-  @Stellar(namespace="STATS", name="SD"
-          , description = "Calculates the standard deviation of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics "
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The standard deviation of the values in the window or NaN if the statistics object is null.")
-  public static class StandardDeviation extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getStandardDeviation() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the sum of logs.
-   */
-  @Stellar(namespace="STATS", name="SUM_LOGS"
-          , description = "Calculates the sum of the (natural) log of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics "
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The sum of the (natural) log of the values in the window or NaN if the statistics object is null.")
-  public static class SumLogs extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getSumLogs() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the sum of squares.
-   */
-  @Stellar(namespace="STATS", name="SUM_SQUARES"
-          , description = "Calculates the sum of the squares of the accumulated values (or in the window if a window is used)."
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The sum of the squares of the values in the window or NaN if the statistics object is null.")
-  public static class SumSquares extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getSumSquares() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the kurtosis.
-   */
-  @Stellar(namespace="STATS", name="KURTOSIS"
-          , description = "Calculates the kurtosis of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics "
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The kurtosis of the values in the window or NaN if the statistics object is null.")
-  public static class Kurtosis extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getKurtosis() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the skewness.
-   */
-  @Stellar(namespace="STATS", name="SKEWNESS"
-          , description = "Calculates the skewness of the accumulated values (or in the window if a window is used).  See http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics "
-          , params = {
-            "stats - The Stellar statistics object"
-                     }
-          , returns = "The skewness of the values in the window or NaN if the statistics object is null.")
-  public static class Skewness extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      return (stats != null) ? stats.getSkewness() : Double.NaN;
-    }
-  }
-
-  /**
-   * Calculates the Pth percentile.
-   *
-   * STATS_PERCENTILE(stats, 0.90)
-   */
-  @Stellar(namespace="STATS", name="PERCENTILE"
-          , description = "Computes the p'th percentile of the accumulated values (or in the window if a window is used)."
-          , params = {
-          "stats - The Stellar statistics object"
-          ,"p - a double where 0 <= p < 1 representing the percentile"
-
-                     }
-          , returns = "The p'th percentile of the data or NaN if the statistics object is null"
-          )
-  public static class Percentile extends BaseStellarFunction {
-    @Override
-    public Object apply(List<Object> args) {
-      StatisticsProvider stats = convert(args.get(0), StatisticsProvider.class);
-      Double p = convert(args.get(1), Double.class);
-
-      Double result;
-      if(stats == null || p == null) {
-        result = Double.NaN;
-      } else {
-        result = stats.getPercentile(p);
-      }
-
-      return result;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/OnlineStatisticsProvider.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/OnlineStatisticsProvider.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/OnlineStatisticsProvider.java
deleted file mode 100644
index 06a5b81..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/OnlineStatisticsProvider.java
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- *
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-
-package org.apache.metron.common.math.stats;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.KryoSerializable;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import com.tdunning.math.stats.AVLTreeDigest;
-import com.tdunning.math.stats.TDigest;
-import org.apache.commons.math3.util.FastMath;
-
-import java.nio.ByteBuffer;
-
-/**
- * A (near) constant memory implementation of a statistics provider.
- * For first order statistics, simple terms are stored and composed
- * to return the statistics results.  This is intended to provide a
- * mergeable implementation for a statistics provider.
- */
-public class OnlineStatisticsProvider implements StatisticsProvider, KryoSerializable {
-  /**
-   * A sensible default for compression to use in the T-Digest.
-   * As per https://github.com/tdunning/t-digest/blob/master/src/main/java/com/tdunning/math/stats/TDigest.java#L86
-   * 100 is a sensible default and the number of centroids retained (to construct the sketch)
-   * is usually a smallish (usually < 10) multiple of the compression.  However, we have found through some
-   * testing that 150 gives a bit finer granularity for smaller numbers of data points.
-   */
-  public static final int COMPRESSION = 150;
-
-
-  /**
-   * A distributional sketch that uses a variant of 1-D k-means to construct a tree of ranges
-   * that sketches the distribution.  See https://github.com/tdunning/t-digest#t-digest for
-   * more detail.
-   */
-  private TDigest digest;
-
-  private long n = 0;
-  private double sum = 0;
-  private double sumOfSquares = 0;
-  private double sumOfLogs = 0;
-  private Double min = null;
-  private Double max = null;
-
-  //\mu_1, E[X]
-  private double M1 = 0;
-  //\mu_2: E[(X - \mu)^2]
-  private double M2 = 0;
-  //\mu_3: E[(X - \mu)^3]
-  private double M3 = 0;
-  //\mu_4: E[(X - \mu)^4]
-  private double M4 = 0;
-
-  public OnlineStatisticsProvider() {
-    digest = TDigest.createAvlTreeDigest(COMPRESSION);
-  }
-
-  /**
-   * Add a value.
-   * NOTE: This does not store the point, but only updates internal state.
-   * NOTE: This is NOT threadsafe.
-   * @param value
-   */
-  @Override
-  public void addValue(double value) {
-    long n1 = n;
-    min = min == null?value:Math.min(min, value);
-    max = max == null?value:Math.max(max, value);
-    sum += value;
-    sumOfLogs += Math.log(value);
-    sumOfSquares += value*value;
-    digest.add(value);
-    n++;
-    double delta, delta_n, delta_n2, term1;
-    //delta between the value and the mean
-    delta = value - M1;
-    //(x - E[x])/n
-    delta_n = delta / n;
-    delta_n2 = delta_n * delta_n;
-    term1 = delta * delta_n * n1;
-
-    // Adjusting expected value: See Knuth TAOCP vol 2, 3rd edition, page 232
-    M1 += delta_n;
-    // Adjusting the \mu_i, see http://www.johndcook.com/blog/skewness_kurtosis/
-    M4 += term1 * delta_n2 * (n*n - 3*n + 3) + 6 * delta_n2 * M2 - 4 * delta_n * M3;
-    M3 += term1 * delta_n * (n - 2) - 3 * delta_n * M2;
-    M2 += term1;
-    checkFlowError(sumOfSquares, sum, sumOfSquares, M1, M2, M3, M4);
-
-  }
-
-  private void checkFlowError(double sumOfSquares, double sum, double... vals) {
-    //overflow
-    for(double val : vals) {
-      if(Double.isInfinite(val)) {
-        throw new IllegalStateException("Double overflow!");
-      }
-    }
-    //underflow.  It is sufficient to check sumOfSquares because sumOfSquares is going to converge to 0 faster than sum
-    //in the situation where we're looking at an underflow.
-    if(sumOfSquares == 0.0 && sum > 0) {
-      throw new IllegalStateException("Double overflow!");
-    }
-  }
-
-  @Override
-  public long getCount() {
-    return n;
-  }
-
-  @Override
-  public double getMin() {
-    return min == null?Double.NaN:min;
-  }
-
-  @Override
-  public double getMax() {
-    return max == null?Double.NaN:max;
-  }
-
-  @Override
-  public double getMean() {
-    return getSum()/getCount();
-  }
-
-  @Override
-  public double getSum() {
-    return sum;
-  }
-
-  @Override
-  public double getVariance() {
-    return M2/(n - 1.0);
-  }
-
-  @Override
-  public double getStandardDeviation() {
-    return FastMath.sqrt(getVariance());
-  }
-
-  @Override
-  public double getGeometricMean() {
-    throw new UnsupportedOperationException("Unwilling to compute the geometric mean.");
-  }
-
-  @Override
-  public double getPopulationVariance() {
-    throw new UnsupportedOperationException("Unwilling to compute the geometric mean.");
-  }
-
-  @Override
-  public double getQuadraticMean() {
-    return FastMath.sqrt(sumOfSquares/n);
-  }
-
-  @Override
-  public double getSumLogs() {
-    return sumOfLogs;
-  }
-
-  @Override
-  public double getSumSquares() {
-    return sumOfSquares;
-  }
-
-  /**
-   * Unbiased kurtosis.
-   * See http://commons.apache.org/proper/commons-math/apidocs/org/apache/commons/math4/stat/descriptive/moment/Kurtosis.html
-   * @return
-   */
-  @Override
-  public double getKurtosis() {
-    //kurtosis = { [n(n+1) / (n -1)(n - 2)(n-3)] \mu_4 / std^4 } - [3(n-1)^2 / (n-2)(n-3)]
-    if(n < 4) {
-      return Double.NaN;
-    }
-    double std = getStandardDeviation();
-    double t1 = (1.0*n)*(n+1)/((n-1)*(n-2)*(n-3));
-    double t3 = 3.0*((n-1)*(n-1))/((n-2)*(n-3));
-    return t1*(M4/FastMath.pow(std, 4))-t3;
-  }
-
-  /**
-   * Unbiased skewness.
-   * See  http://commons.apache.org/proper/commons-math/apidocs/org/apache/commons/math4/stat/descriptive/moment/Skewness.html
-   * @return
-   */
-  @Override
-  public double getSkewness() {
-    //  skewness = [n / (n -1) (n - 2)] sum[(x_i - mean)^3] / std^3
-    if(n < 3) {
-      return Double.NaN;
-    }
-    double t1 = (1.0*n)/((n - 1)*(n-2));
-    double std = getStandardDeviation();
-    return t1*M3/FastMath.pow(std, 3);
-  }
-
-  /**
-   * This returns an approximate percentile based on a t-digest.
-   * @param p
-   * @return
-   */
-  @Override
-  public double getPercentile(double p) {
-    return digest.quantile(p/100.0);
-  }
-
-  @Override
-  public StatisticsProvider merge(StatisticsProvider provider) {
-    OnlineStatisticsProvider combined = new OnlineStatisticsProvider();
-    OnlineStatisticsProvider a = this;
-    OnlineStatisticsProvider b = (OnlineStatisticsProvider)provider;
-
-    //Combining the simple terms that obviously form a semigroup
-    combined.n = a.n + b.n;
-    combined.sum = a.sum + b.sum;
-    combined.min = Math.min(a.min, b.min);
-    combined.max = Math.max(a.max, b.max);
-    combined.sumOfSquares = a.sumOfSquares + b.sumOfSquares;
-    combined.sumOfLogs = a.sumOfLogs+ b.sumOfLogs;
-
-    // Adjusting the standardized moments, see http://www.johndcook.com/blog/skewness_kurtosis/
-    double delta = b.M1 - a.M1;
-    double delta2 = delta*delta;
-    double delta3 = delta*delta2;
-    double delta4 = delta2*delta2;
-
-    combined.M1 = (a.n*a.M1 + b.n*b.M1) / combined.n;
-
-    combined.M2 = a.M2 + b.M2 +
-            delta2 * a.n * b.n / combined.n;
-
-    combined.M3 = a.M3 + b.M3 +
-            delta3 * a.n * b.n * (a.n - b.n)/(combined.n*combined.n);
-    combined.M3 += 3.0*delta * (a.n*b.M2 - b.n*a.M2) / combined.n;
-
-    combined.M4 = a.M4 + b.M4 + delta4*a.n*b.n * (a.n*a.n - a.n*b.n + b.n*b.n) /
-            (combined.n*combined.n*combined.n);
-    combined.M4 += 6.0*delta2 * (a.n*a.n*b.M2 + b.n*b.n*a.M2)/(combined.n*combined.n) +
-            4.0*delta*(a.n*b.M3 - b.n*a.M3) / combined.n;
-
-    //Merging the distributional sketches
-    combined.digest.add(a.digest);
-    combined.digest.add(b.digest);
-    checkFlowError(combined.sumOfSquares, sum, combined.sumOfSquares, combined.M1, combined.M2, combined.M3, combined.M4);
-    return combined;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
-
-    OnlineStatisticsProvider that = (OnlineStatisticsProvider) o;
-
-    if (n != that.n) return false;
-    if (Double.compare(that.sum, sum) != 0) return false;
-    if (Double.compare(that.sumOfSquares, sumOfSquares) != 0) return false;
-    if (Double.compare(that.sumOfLogs, sumOfLogs) != 0) return false;
-    if (Double.compare(that.M1, M1) != 0) return false;
-    if (Double.compare(that.M2, M2) != 0) return false;
-    if (Double.compare(that.M3, M3) != 0) return false;
-    if (Double.compare(that.M4, M4) != 0) return false;
-    if (digest != null ? !digest.equals(that.digest) : that.digest != null) return false;
-    if (min != null ? !min.equals(that.min) : that.min != null) return false;
-    return max != null ? max.equals(that.max) : that.max == null;
-
-  }
-
-  @Override
-  public int hashCode() {
-    int result;
-    long temp;
-    result = digest != null ? digest.hashCode() : 0;
-    result = 31 * result + (int) (n ^ (n >>> 32));
-    temp = Double.doubleToLongBits(sum);
-    result = 31 * result + (int) (temp ^ (temp >>> 32));
-    temp = Double.doubleToLongBits(sumOfSquares);
-    result = 31 * result + (int) (temp ^ (temp >>> 32));
-    temp = Double.doubleToLongBits(sumOfLogs);
-    result = 31 * result + (int) (temp ^ (temp >>> 32));
-    result = 31 * result + (min != null ? min.hashCode() : 0);
-    result = 31 * result + (max != null ? max.hashCode() : 0);
-    temp = Double.doubleToLongBits(M1);
-    result = 31 * result + (int) (temp ^ (temp >>> 32));
-    temp = Double.doubleToLongBits(M2);
-    result = 31 * result + (int) (temp ^ (temp >>> 32));
-    temp = Double.doubleToLongBits(M3);
-    result = 31 * result + (int) (temp ^ (temp >>> 32));
-    temp = Double.doubleToLongBits(M4);
-    result = 31 * result + (int) (temp ^ (temp >>> 32));
-    return result;
-  }
-
-  @Override
-  public void write(Kryo kryo, Output output) {
-    //storing tdigest
-    ByteBuffer outBuffer = ByteBuffer.allocate(digest.byteSize());
-    digest.asBytes(outBuffer);
-    byte[] tdigestSerialized = outBuffer.array();
-    output.writeInt(tdigestSerialized.length);
-    output.writeBytes(tdigestSerialized);
-    output.writeLong(n);
-    output.writeDouble(sum);
-    output.writeDouble(sumOfSquares);
-    output.writeDouble(sumOfLogs);
-    output.writeDouble(getMin());
-    output.writeDouble(getMax());
-    output.writeDouble(M1);
-    output.writeDouble(M2);
-    output.writeDouble(M3);
-    output.writeDouble(M4);
-  }
-
-  @Override
-  public void read(Kryo kryo, Input input) {
-    int digestSize = input.readInt();
-    byte[] digestBytes = input.readBytes(digestSize);
-    ByteBuffer digestBuff = ByteBuffer.wrap(digestBytes);
-    digest = AVLTreeDigest.fromBytes(digestBuff);
-    n = input.readLong();
-    sum = input.readDouble();
-    sumOfSquares = input.readDouble();
-    sumOfLogs = input.readDouble();
-    min = input.readDouble();
-    max = input.readDouble();
-    M1 = input.readDouble();
-    M2 = input.readDouble();
-    M3 = input.readDouble();
-    M4 = input.readDouble();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/StatisticsProvider.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/StatisticsProvider.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/StatisticsProvider.java
deleted file mode 100644
index 0561d60..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/StatisticsProvider.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- *
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-package org.apache.metron.common.math.stats;
-
-
-/**
- * Provides statistical functions.
- */
-public interface StatisticsProvider {
-  void addValue(double value);
-  long getCount();
-  double getMin();
-  double getMax();
-  double getMean();
-  double getSum();
-  double getVariance();
-  double getStandardDeviation();
-  double getGeometricMean();
-  double getPopulationVariance();
-  double getQuadraticMean();
-  double getSumLogs();
-  double getSumSquares();
-
-  /**
-   * Unbiased Kurtosis.
-   * See http://commons.apache.org/proper/commons-math/apidocs/org/apache/commons/math4/stat/descriptive/moment/Kurtosis.html
-   * @return unbiased kurtosis
-   */
-  double getKurtosis();
-
-  /**
-   * Unbiased skewness.
-   * See  http://commons.apache.org/proper/commons-math/apidocs/org/apache/commons/math4/stat/descriptive/moment/Skewness.html
-   * @return
-   */
-  double getSkewness();
-
-  double getPercentile(double p);
-
-  /**
-   * Merge an existing statistics provider.
-   * @param provider The provider to merge with the current object
-   * @return A merged statistics provider.
-   */
-  StatisticsProvider merge(StatisticsProvider provider);
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/WindowedStatisticsProvider.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/WindowedStatisticsProvider.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/WindowedStatisticsProvider.java
deleted file mode 100644
index fc145ac..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/math/stats/WindowedStatisticsProvider.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- *
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-package org.apache.metron.common.math.stats;
-
-import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
-
-/**
- * Provides basic summary statistics to Stellar.
- *
- * Used as an adapter to provide an interface to the underlying Commons
- * Math class that provide summary statistics for windows of data.
- * NOTE: Windowed statistics providers cannot be merged.
- */
-public class WindowedStatisticsProvider implements StatisticsProvider {
-
-  /**
-   * DescriptiveStatistics stores a rolling window of input data elements
-   * which is then used to calculate the summary statistic.  There are some
-   * summary statistics like kurtosis and percentiles, that can only
-   * be calculated with this.
-   *
-   * This implementation is used if the windowSize > 0.
-   */
-  private DescriptiveStatistics descStats;
-
-  public WindowedStatisticsProvider(int windowSize) {
-    descStats = new DescriptiveStatistics(windowSize);
-  }
-
-  @Override
-  public void addValue(double value) {
-    descStats.addValue(value);
-  }
-
-  @Override
-  public long getCount() {
-    return descStats.getN();
-  }
-
-  @Override
-  public double getMin() {
-    return descStats.getMin();
-  }
-
-  @Override
-  public double getMax() {
-    return descStats.getMax();
-  }
-
-  @Override
-  public double getMean() {
-    return descStats.getMean();
-  }
-
-  @Override
-  public double getSum() {
-    return descStats.getSum();
-  }
-
-  @Override
-  public double getVariance() {
-    return descStats.getVariance();
-  }
-
-  @Override
-  public double getStandardDeviation() {
-    return descStats.getStandardDeviation();
-  }
-
-  @Override
-  public double getGeometricMean() {
-    return descStats.getGeometricMean();
-  }
-
-  @Override
-  public double getPopulationVariance() {
-    return descStats.getPopulationVariance();
-  }
-
-  @Override
-  public double getQuadraticMean() {
-    return descStats.getQuadraticMean();
-  }
-
-  @Override
-  public double getSumLogs() {
-    throw new UnsupportedOperationException("sum logs not available if 'windowSize' > 0");
-  }
-
-  @Override
-  public double getSumSquares() {
-    return descStats.getSumsq();
-  }
-
-  @Override
-  public double getKurtosis() {
-    return descStats.getKurtosis();
-  }
-
-  @Override
-  public double getSkewness() {
-    return descStats.getSkewness();
-  }
-
-  @Override
-  public double getPercentile(double p) {
-    return descStats.getPercentile(p);
-  }
-
-  @Override
-  public StatisticsProvider merge(StatisticsProvider provider) {
-    throw new UnsupportedOperationException("Windowed Statistics cannot be merged.");
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
-
-    WindowedStatisticsProvider that = (WindowedStatisticsProvider) o;
-
-    return descStats != null ? descStats.equals(that.descStats) : that.descStats == null;
-
-  }
-
-  @Override
-  public int hashCode() {
-    return descStats != null ? descStats.hashCode() : 0;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b1c89df6/metron-platform/metron-common/src/test/java/org/apache/metron/common/math/stats/OnlineStatisticsProviderTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/math/stats/OnlineStatisticsProviderTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/math/stats/OnlineStatisticsProviderTest.java
deleted file mode 100644
index 06339b7..0000000
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/math/stats/OnlineStatisticsProviderTest.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- *
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-package org.apache.metron.common.math.stats;
-
-import org.apache.commons.math.random.GaussianRandomGenerator;
-import org.apache.commons.math.random.MersenneTwister;
-import org.apache.commons.math.random.RandomGenerator;
-import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
-import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class OnlineStatisticsProviderTest {
-
-  public static void validateStatisticsProvider( StatisticsProvider statsProvider
-                                               , SummaryStatistics summaryStats
-                                               , DescriptiveStatistics stats
-                                               ) {
-    //N
-    Assert.assertEquals(statsProvider.getCount(), stats.getN());
-    //sum
-    Assert.assertEquals(statsProvider.getSum(), stats.getSum(), 1e-3);
-    //sum of squares
-    Assert.assertEquals(statsProvider.getSumSquares(), stats.getSumsq(), 1e-3);
-    //sum of squares
-    Assert.assertEquals(statsProvider.getSumLogs(), summaryStats.getSumOfLogs(), 1e-3);
-    //Mean
-    Assert.assertEquals(statsProvider.getMean(), stats.getMean(), 1e-3);
-    //Quadratic Mean
-    Assert.assertEquals(statsProvider.getQuadraticMean(), summaryStats.getQuadraticMean(), 1e-3);
-    //SD
-    Assert.assertEquals(statsProvider.getStandardDeviation(), stats.getStandardDeviation(), 1e-3);
-    //Variance
-    Assert.assertEquals(statsProvider.getVariance(), stats.getVariance(), 1e-3);
-    //Min
-    Assert.assertEquals(statsProvider.getMin(), stats.getMin(), 1e-3);
-    //Max
-    Assert.assertEquals(statsProvider.getMax(), stats.getMax(), 1e-3);
-
-    //Kurtosis
-    Assert.assertEquals(stats.getKurtosis(), statsProvider.getKurtosis(), 1e-3);
-
-    //Skewness
-    Assert.assertEquals(stats.getSkewness(), statsProvider.getSkewness(), 1e-3);
-    for(double d = 10.0;d < 100.0;d+=10) {
-      //This is a sketch, so we're a bit more forgiving here in our choice of \epsilon.
-      Assert.assertEquals("Percentile mismatch for " + d +"th %ile"
-                         , statsProvider.getPercentile(d)
-                         , stats.getPercentile(d)
-                         , 1e-2
-                         );
-    }
-  }
-
-  private void validateEquality(Iterable<Double> values) {
-    DescriptiveStatistics stats = new DescriptiveStatistics();
-    SummaryStatistics summaryStats = new SummaryStatistics();
-    OnlineStatisticsProvider statsProvider = new OnlineStatisticsProvider();
-    //Test that the aggregated provider gives the same results as the provider that is shown all the data.
-    List<OnlineStatisticsProvider> providers = new ArrayList<>();
-    for(int i = 0;i < 10;++i) {
-      providers.add(new OnlineStatisticsProvider());
-    }
-    int i = 0;
-    for(double d : values) {
-      i++;
-      stats.addValue(d);
-      summaryStats.addValue(d);
-      providers.get(i % providers.size()).addValue(d);
-      statsProvider.addValue(d);
-    }
-    StatisticsProvider aggregatedProvider = providers.get(0);
-    for(int j = 1;j < providers.size();++j) {
-      aggregatedProvider = aggregatedProvider.merge(providers.get(j));
-    }
-    validateStatisticsProvider(statsProvider, summaryStats, stats);
-    validateStatisticsProvider(aggregatedProvider, summaryStats, stats);
-  }
-
-  @Test(expected = IllegalStateException.class)
-  public void testOverflow() {
-    OnlineStatisticsProvider statsProvider = new OnlineStatisticsProvider();
-    statsProvider.addValue(Double.MAX_VALUE + 1);
-  }
-
-  @Test(expected = IllegalStateException.class)
-  public void testUnderflow() {
-    OnlineStatisticsProvider statsProvider = new OnlineStatisticsProvider();
-    double d = 3e-305;
-    for(int i = 0;i < 5;++i,d/=100000) {
-      statsProvider.addValue(d);
-    }
-  }
-
-  @Test
-  public void testNormallyDistributedRandomData() {
-    List<Double> values = new ArrayList<>();
-    GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(0L));
-    for(int i = 0;i < 1000000;++i) {
-      double d = gaussian.nextNormalizedDouble();
-      values.add(d);
-    }
-    validateEquality(values);
-  }
-  @Test
-  public void testNormallyDistributedRandomDataShifted() {
-    List<Double> values = new ArrayList<>();
-    GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(0L));
-    for(int i = 0;i < 1000000;++i) {
-      double d = gaussian.nextNormalizedDouble() + 10;
-      values.add(d);
-    }
-    validateEquality(values);
-  }
-
-  @Test
-  public void testNormallyDistributedRandomDataShiftedBackwards() {
-    List<Double> values = new ArrayList<>();
-    GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(0L));
-    for(int i = 0;i < 1000000;++i) {
-      double d = gaussian.nextNormalizedDouble() - 10;
-      values.add(d);
-    }
-    validateEquality(values);
-  }
-  @Test
-  public void testNormallyDistributedRandomDataSkewed() {
-    List<Double> values = new ArrayList<>();
-    GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(0L));
-    for(int i = 0;i < 1000000;++i) {
-      double d = (gaussian.nextNormalizedDouble()+ 10000) /1000;
-      values.add(d);
-    }
-    validateEquality(values);
-  }
-
-  @Test
-  public void testNormallyDistributedRandomDataAllNegative() {
-    List<Double> values = new ArrayList<>();
-    GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(0L));
-    for(int i = 0;i < 1000000;++i) {
-      double d = -1*gaussian.nextNormalizedDouble();
-      values.add(d);
-    }
-    validateEquality(values);
-  }
-  @Test
-  public void testUniformlyDistributedRandomData() {
-    List<Double> values = new ArrayList<>();
-    for(int i = 0;i < 100000;++i) {
-      double d = Math.random();
-      values.add(d);
-    }
-    validateEquality(values);
-  }
-}