You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2017/09/27 22:03:08 UTC
[2/6] ambari git commit: AMBARI-22077 : Create maven module and
package structure for the anomaly detection engine. (Commit 2) (avijayan)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java
deleted file mode 100644
index 6485ebb..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.alertservice.prototype.testing.utilities;
-
-import org.apache.ambari.metrics.alertservice.prototype.core.MetricsCollectorInterface;
-import org.apache.ambari.metrics.alertservice.prototype.core.RFunctionInvoker;
-import org.apache.ambari.metrics.alertservice.prototype.common.DataSeries;
-import org.apache.ambari.metrics.alertservice.prototype.common.ResultSet;
-import org.apache.ambari.metrics.alertservice.seriesgenerator.MetricSeriesGeneratorFactory;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TreeMap;
-
-public class MetricAnomalyTester {
-
- public static String appId = MetricsCollectorInterface.serviceName;
- static final Log LOG = LogFactory.getLog(MetricAnomalyTester.class);
- static Map<String, TimelineMetric> timelineMetricMap = new HashMap<>();
-
- public static TimelineMetrics runTestAnomalyRequest(MetricAnomalyDetectorTestInput input) throws UnknownHostException {
-
- long currentTime = System.currentTimeMillis();
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- String hostname = InetAddress.getLocalHost().getHostName();
-
- //Train data
- TimelineMetric metric1 = new TimelineMetric();
- if (StringUtils.isNotEmpty(input.getTrainDataName())) {
- metric1 = timelineMetricMap.get(input.getTrainDataName());
- if (metric1 == null) {
- metric1 = new TimelineMetric();
- double[] trainSeries = MetricSeriesGeneratorFactory.generateSeries(input.getTrainDataType(), input.getTrainDataSize(), input.getTrainDataConfigs());
- metric1.setMetricName(input.getTrainDataName());
- metric1.setAppId(appId);
- metric1.setHostName(hostname);
- metric1.setStartTime(currentTime);
- metric1.setInstanceId(null);
- metric1.setMetricValues(getAsTimeSeries(currentTime, trainSeries));
- timelineMetricMap.put(input.getTrainDataName(), metric1);
- }
- timelineMetrics.getMetrics().add(metric1);
- } else {
- LOG.error("No train data name specified");
- }
-
- //Test data
- TimelineMetric metric2 = new TimelineMetric();
- if (StringUtils.isNotEmpty(input.getTestDataName())) {
- metric2 = timelineMetricMap.get(input.getTestDataName());
- if (metric2 == null) {
- metric2 = new TimelineMetric();
- double[] testSeries = MetricSeriesGeneratorFactory.generateSeries(input.getTestDataType(), input.getTestDataSize(), input.getTestDataConfigs());
- metric2.setMetricName(input.getTestDataName());
- metric2.setAppId(appId);
- metric2.setHostName(hostname);
- metric2.setStartTime(currentTime);
- metric2.setInstanceId(null);
- metric2.setMetricValues(getAsTimeSeries(currentTime, testSeries));
- timelineMetricMap.put(input.getTestDataName(), metric2);
- }
- timelineMetrics.getMetrics().add(metric2);
- } else {
- LOG.warn("No test data name specified");
- }
-
- //Invoke method
- if (CollectionUtils.isNotEmpty(input.getMethods())) {
- RFunctionInvoker.setScriptsDir("/etc/ambari-metrics-collector/conf/R-scripts");
- for (String methodType : input.getMethods()) {
- ResultSet result = RFunctionInvoker.executeMethod(methodType, getAsDataSeries(metric1), getAsDataSeries(metric2), input.getMethodConfigs());
- TimelineMetric timelineMetric = getAsTimelineMetric(result, methodType, input, currentTime, hostname);
- if (timelineMetric != null) {
- timelineMetrics.getMetrics().add(timelineMetric);
- }
- }
- } else {
- LOG.warn("No anomaly method requested");
- }
-
- return timelineMetrics;
- }
-
-
- private static TimelineMetric getAsTimelineMetric(ResultSet result, String methodType, MetricAnomalyDetectorTestInput input, long currentTime, String hostname) {
-
- if (result == null) {
- return null;
- }
-
- TimelineMetric timelineMetric = new TimelineMetric();
- if (methodType.equals("tukeys") || methodType.equals("ema")) {
- timelineMetric.setMetricName(input.getTrainDataName() + "_" + input.getTestDataName() + "_" + methodType + "_" + currentTime);
- timelineMetric.setHostName(hostname);
- timelineMetric.setAppId(appId);
- timelineMetric.setInstanceId(null);
- timelineMetric.setStartTime(currentTime);
-
- TreeMap<Long, Double> metricValues = new TreeMap<>();
- if (result.resultset.size() > 0) {
- double[] ts = result.resultset.get(0);
- double[] metrics = result.resultset.get(1);
- for (int i = 0; i < ts.length; i++) {
- if (i == 0) {
- timelineMetric.setStartTime((long) ts[i]);
- }
- metricValues.put((long) ts[i], metrics[i]);
- }
- }
- timelineMetric.setMetricValues(metricValues);
- return timelineMetric;
- }
- return null;
- }
-
-
- private static TreeMap<Long, Double> getAsTimeSeries(long currentTime, double[] values) {
-
- long startTime = currentTime - (values.length - 1) * 60 * 1000;
- TreeMap<Long, Double> metricValues = new TreeMap<>();
-
- for (int i = 0; i < values.length; i++) {
- metricValues.put(startTime, values[i]);
- startTime += (60 * 1000);
- }
- return metricValues;
- }
-
- private static DataSeries getAsDataSeries(TimelineMetric timelineMetric) {
-
- TreeMap<Long, Double> metricValues = timelineMetric.getMetricValues();
- double[] timestamps = new double[metricValues.size()];
- double[] values = new double[metricValues.size()];
- int i = 0;
-
- for (Long timestamp : metricValues.keySet()) {
- timestamps[i] = timestamp;
- values[i++] = metricValues.get(timestamp);
- }
- return new DataSeries(timelineMetric.getMetricName() + "_" + timelineMetric.getAppId() + "_" + timelineMetric.getHostName(), timestamps, values);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java
deleted file mode 100644
index b817f3e..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.testing.utilities;
-
-/**
- * Class which was originally used to send test series from AMS to Spark through Kafka.
- */
-
-public class TestMetricSeriesGenerator {
- //implements Runnable {
-
-// private Map<TestSeriesInputRequest, AbstractMetricSeries> configuredSeries = new HashMap<>();
-// private static final Log LOG = LogFactory.getLog(TestMetricSeriesGenerator.class);
-// private TimelineMetricStore metricStore;
-// private String hostname;
-//
-// public TestMetricSeriesGenerator(TimelineMetricStore metricStore) {
-// this.metricStore = metricStore;
-// try {
-// this.hostname = InetAddress.getLocalHost().getHostName();
-// } catch (UnknownHostException e) {
-// e.printStackTrace();
-// }
-// }
-//
-// public void addSeries(TestSeriesInputRequest inputRequest) {
-// if (!configuredSeries.containsKey(inputRequest)) {
-// AbstractMetricSeries metricSeries = MetricSeriesGeneratorFactory.generateSeries(inputRequest.getSeriesType(), inputRequest.getConfigs());
-// configuredSeries.put(inputRequest, metricSeries);
-// LOG.info("Added series " + inputRequest.getSeriesName());
-// }
-// }
-//
-// public void removeSeries(String seriesName) {
-// boolean isPresent = false;
-// TestSeriesInputRequest tbd = null;
-// for (TestSeriesInputRequest inputRequest : configuredSeries.keySet()) {
-// if (inputRequest.getSeriesName().equals(seriesName)) {
-// isPresent = true;
-// tbd = inputRequest;
-// }
-// }
-// if (isPresent) {
-// LOG.info("Removing series " + seriesName);
-// configuredSeries.remove(tbd);
-// } else {
-// LOG.info("Series not found : " + seriesName);
-// }
-// }
-//
-// @Override
-// public void run() {
-// long currentTime = System.currentTimeMillis();
-// TimelineMetrics timelineMetrics = new TimelineMetrics();
-//
-// for (TestSeriesInputRequest input : configuredSeries.keySet()) {
-// AbstractMetricSeries metricSeries = configuredSeries.get(input);
-// TimelineMetric timelineMetric = new TimelineMetric();
-// timelineMetric.setMetricName(input.getSeriesName());
-// timelineMetric.setAppId("anomaly-engine-test-metric");
-// timelineMetric.setInstanceId(null);
-// timelineMetric.setStartTime(currentTime);
-// timelineMetric.setHostName(hostname);
-// TreeMap<Long, Double> metricValues = new TreeMap();
-// metricValues.put(currentTime, metricSeries.nextValue());
-// timelineMetric.setMetricValues(metricValues);
-// timelineMetrics.addOrMergeTimelineMetric(timelineMetric);
-// LOG.info("Emitting metric with appId = " + timelineMetric.getAppId());
-// }
-// try {
-// LOG.info("Publishing test metrics for " + timelineMetrics.getMetrics().size() + " series.");
-// metricStore.putMetrics(timelineMetrics);
-// } catch (Exception e) {
-// LOG.error(e);
-// }
-// }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java
deleted file mode 100644
index a424f8e..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.testing.utilities;
-
-import org.apache.htrace.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper;
-
-import javax.xml.bind.annotation.XmlRootElement;
-import java.util.Collections;
-import java.util.Map;
-
-@XmlRootElement
-public class TestSeriesInputRequest {
-
- private String seriesName;
- private String seriesType;
- private Map<String, String> configs;
-
- public TestSeriesInputRequest() {
- }
-
- public TestSeriesInputRequest(String seriesName, String seriesType, Map<String, String> configs) {
- this.seriesName = seriesName;
- this.seriesType = seriesType;
- this.configs = configs;
- }
-
- public String getSeriesName() {
- return seriesName;
- }
-
- public void setSeriesName(String seriesName) {
- this.seriesName = seriesName;
- }
-
- public String getSeriesType() {
- return seriesType;
- }
-
- public void setSeriesType(String seriesType) {
- this.seriesType = seriesType;
- }
-
- public Map<String, String> getConfigs() {
- return configs;
- }
-
- public void setConfigs(Map<String, String> configs) {
- this.configs = configs;
- }
-
- @Override
- public boolean equals(Object o) {
- TestSeriesInputRequest anotherInput = (TestSeriesInputRequest)o;
- return anotherInput.getSeriesName().equals(this.getSeriesName());
- }
-
- @Override
- public int hashCode() {
- return seriesName.hashCode();
- }
-
- public static void main(String[] args) {
-
- ObjectMapper objectMapper = new ObjectMapper();
- TestSeriesInputRequest testSeriesInputRequest = new TestSeriesInputRequest("test", "ema", Collections.singletonMap("key","value"));
- try {
- System.out.print(objectMapper.writeValueAsString(testSeriesInputRequest));
- } catch (JsonProcessingException e) {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java
deleted file mode 100644
index a8e31bf..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-public interface AbstractMetricSeries {
-
- public double nextValue();
- public double[] getSeries(int n);
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java
deleted file mode 100644
index 4158ff4..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class DualBandMetricSeries implements AbstractMetricSeries {
-
- double lowBandValue = 0.0;
- double lowBandDeviationPercentage = 0.0;
- int lowBandPeriodSize = 10;
- double highBandValue = 1.0;
- double highBandDeviationPercentage = 0.0;
- int highBandPeriodSize = 10;
-
- Random random = new Random();
- double lowBandValueLowerLimit, lowBandValueHigherLimit;
- double highBandLowerLimit, highBandUpperLimit;
- int l = 0, h = 0;
-
- public DualBandMetricSeries(double lowBandValue,
- double lowBandDeviationPercentage,
- int lowBandPeriodSize,
- double highBandValue,
- double highBandDeviationPercentage,
- int highBandPeriodSize) {
- this.lowBandValue = lowBandValue;
- this.lowBandDeviationPercentage = lowBandDeviationPercentage;
- this.lowBandPeriodSize = lowBandPeriodSize;
- this.highBandValue = highBandValue;
- this.highBandDeviationPercentage = highBandDeviationPercentage;
- this.highBandPeriodSize = highBandPeriodSize;
- init();
- }
-
- private void init() {
- lowBandValueLowerLimit = lowBandValue - lowBandDeviationPercentage * lowBandValue;
- lowBandValueHigherLimit = lowBandValue + lowBandDeviationPercentage * lowBandValue;
- highBandLowerLimit = highBandValue - highBandDeviationPercentage * highBandValue;
- highBandUpperLimit = highBandValue + highBandDeviationPercentage * highBandValue;
- }
-
- @Override
- public double nextValue() {
-
- double value = 0.0;
-
- if (l < lowBandPeriodSize) {
- value = lowBandValueLowerLimit + (lowBandValueHigherLimit - lowBandValueLowerLimit) * random.nextDouble();
- l++;
- } else if (h < highBandPeriodSize) {
- value = highBandLowerLimit + (highBandUpperLimit - highBandLowerLimit) * random.nextDouble();
- h++;
- }
-
- if (l == lowBandPeriodSize && h == highBandPeriodSize) {
- l = 0;
- h = 0;
- }
-
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
- double[] series = new double[n];
- for (int i = 0; i < n; i++) {
- series[i] = nextValue();
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java
deleted file mode 100644
index 1e37ff3..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Random;
-
-public class MetricSeriesGeneratorFactory {
-
- /**
- * Return a normally distributed data series with some deviation % and outliers.
- *
- * @param n size of the data series
- * @param value The value around which the uniform data series is centered on.
- * @param deviationPercentage The allowed deviation % on either side of the uniform value. For example, if value = 10, and deviation % is 0.1, the series values lie between 0.9 to 1.1.
- * @param outlierProbability The probability of finding an outlier in the series.
- * @param outlierDeviationLowerPercentage min percentage outlier should be away from the uniform value in % terms. if value = 10 and outlierDeviationPercentage = 30%, the outlier is 7 and 13.
- * @param outlierDeviationHigherPercentage max percentage outlier should be away from the uniform value in % terms. if value = 10 and outlierDeviationPercentage = 60%, the outlier is 4 and 16.
- * @param outliersAboveValue Outlier should be greater or smaller than the value.
- * @return uniform series
- */
- public static double[] createUniformSeries(int n,
- double value,
- double deviationPercentage,
- double outlierProbability,
- double outlierDeviationLowerPercentage,
- double outlierDeviationHigherPercentage,
- boolean outliersAboveValue) {
-
- UniformMetricSeries metricSeries = new UniformMetricSeries(value,
- deviationPercentage,
- outlierProbability,
- outlierDeviationLowerPercentage,
- outlierDeviationHigherPercentage,
- outliersAboveValue);
-
- return metricSeries.getSeries(n);
- }
-
-
- /**
- * /**
- * Returns a normally distributed series.
- *
- * @param n size of the data series
- * @param mean mean of the distribution
- * @param sd sd of the distribution
- * @param outlierProbability sd of the distribution
- * @param outlierDeviationSDTimesLower Lower Limit of the outlier with respect to times sdev from the mean.
- * @param outlierDeviationSDTimesHigher Higher Limit of the outlier with respect to times sdev from the mean.
- * @param outlierOnRightEnd Outlier should be on the right end or the left end.
- * @return normal series
- */
- public static double[] createNormalSeries(int n,
- double mean,
- double sd,
- double outlierProbability,
- double outlierDeviationSDTimesLower,
- double outlierDeviationSDTimesHigher,
- boolean outlierOnRightEnd) {
-
-
- NormalMetricSeries metricSeries = new NormalMetricSeries(mean,
- sd,
- outlierProbability,
- outlierDeviationSDTimesLower,
- outlierDeviationSDTimesHigher,
- outlierOnRightEnd);
-
- return metricSeries.getSeries(n);
- }
-
-
- /**
- * Returns a monotonically increasing / decreasing series
- *
- * @param n size of the data series
- * @param startValue Start value of the monotonic sequence
- * @param slope direction of monotonicity m > 0 for increasing and m < 0 for decreasing.
- * @param deviationPercentage The allowed deviation % on either side of the current 'y' value. For example, if current value = 10 according to slope, and deviation % is 0.1, the series values lie between 0.9 to 1.1.
- * @param outlierProbability The probability of finding an outlier in the series.
- * @param outlierDeviationLowerPercentage min percentage outlier should be away from the current 'y' value in % terms. if value = 10 and outlierDeviationPercentage = 30%, the outlier is 7 and 13.
- * @param outlierDeviationHigherPercentage max percentage outlier should be away from the current 'y' value in % terms. if value = 10 and outlierDeviationPercentage = 60%, the outlier is 4 and 16.
- * @param outliersAboveValue Outlier should be greater or smaller than the 'y' value.
- * @return
- */
- public static double[] createMonotonicSeries(int n,
- double startValue,
- double slope,
- double deviationPercentage,
- double outlierProbability,
- double outlierDeviationLowerPercentage,
- double outlierDeviationHigherPercentage,
- boolean outliersAboveValue) {
-
- MonotonicMetricSeries metricSeries = new MonotonicMetricSeries(startValue,
- slope,
- deviationPercentage,
- outlierProbability,
- outlierDeviationLowerPercentage,
- outlierDeviationHigherPercentage,
- outliersAboveValue);
-
- return metricSeries.getSeries(n);
- }
-
-
- /**
- * Returns a dual band series (lower and higher)
- *
- * @param n size of the data series
- * @param lowBandValue lower band value
- * @param lowBandDeviationPercentage lower band deviation
- * @param lowBandPeriodSize lower band
- * @param highBandValue high band centre value
- * @param highBandDeviationPercentage high band deviation.
- * @param highBandPeriodSize high band size
- * @return
- */
- public static double[] getDualBandSeries(int n,
- double lowBandValue,
- double lowBandDeviationPercentage,
- int lowBandPeriodSize,
- double highBandValue,
- double highBandDeviationPercentage,
- int highBandPeriodSize) {
-
- DualBandMetricSeries metricSeries = new DualBandMetricSeries(lowBandValue,
- lowBandDeviationPercentage,
- lowBandPeriodSize,
- highBandValue,
- highBandDeviationPercentage,
- highBandPeriodSize);
-
- return metricSeries.getSeries(n);
- }
-
- /**
- * Returns a step function series.
- *
- * @param n size of the data series
- * @param startValue start steady value
- * @param steadyValueDeviationPercentage required devation in the steady state value
- * @param steadyPeriodSlope direction of monotonicity m > 0 for increasing and m < 0 for decreasing, m = 0 no increase or decrease.
- * @param steadyPeriodMinSize min size for step period
- * @param steadyPeriodMaxSize max size for step period.
- * @param stepChangePercentage Increase / decrease in steady state to denote a step in terms of deviation percentage from the last value.
- * @param upwardStep upward or downward step.
- * @return
- */
- public static double[] getStepFunctionSeries(int n,
- double startValue,
- double steadyValueDeviationPercentage,
- double steadyPeriodSlope,
- int steadyPeriodMinSize,
- int steadyPeriodMaxSize,
- double stepChangePercentage,
- boolean upwardStep) {
-
- StepFunctionMetricSeries metricSeries = new StepFunctionMetricSeries(startValue,
- steadyValueDeviationPercentage,
- steadyPeriodSlope,
- steadyPeriodMinSize,
- steadyPeriodMaxSize,
- stepChangePercentage,
- upwardStep);
-
- return metricSeries.getSeries(n);
- }
-
- /**
- * Series with small period of turbulence and then back to steady.
- *
- * @param n size of the data series
- * @param steadyStateValue steady state center value
- * @param steadyStateDeviationPercentage steady state deviation in percentage
- * @param turbulentPeriodDeviationLowerPercentage turbulent state lower limit in terms of percentage from centre value.
- * @param turbulentPeriodDeviationHigherPercentage turbulent state higher limit in terms of percentage from centre value.
- * @param turbulentPeriodLength turbulent period length (number of points)
- * @param turbulentStatePosition Where the turbulent state should be 0 - at the beginning, 1 - in the middle (25% - 50% of the series), 2 - at the end of the series.
- * @return
- */
- public static double[] getSteadySeriesWithTurbulentPeriod(int n,
- double steadyStateValue,
- double steadyStateDeviationPercentage,
- double turbulentPeriodDeviationLowerPercentage,
- double turbulentPeriodDeviationHigherPercentage,
- int turbulentPeriodLength,
- int turbulentStatePosition
- ) {
-
-
- SteadyWithTurbulenceMetricSeries metricSeries = new SteadyWithTurbulenceMetricSeries(n,
- steadyStateValue,
- steadyStateDeviationPercentage,
- turbulentPeriodDeviationLowerPercentage,
- turbulentPeriodDeviationHigherPercentage,
- turbulentPeriodLength,
- turbulentStatePosition);
-
- return metricSeries.getSeries(n);
- }
-
-
- public static double[] generateSeries(String type, int n, Map<String, String> configs) {
-
- double[] series;
- switch (type) {
-
- case "normal":
- series = createNormalSeries(n,
- Double.parseDouble(configs.getOrDefault("mean", "0")),
- Double.parseDouble(configs.getOrDefault("sd", "1")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesLower", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesHigher", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outlierOnRightEnd", "true")));
- break;
-
- case "uniform":
- series = createUniformSeries(n,
- Double.parseDouble(configs.getOrDefault("value", "10")),
- Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
- break;
-
- case "monotonic":
- series = createMonotonicSeries(n,
- Double.parseDouble(configs.getOrDefault("startValue", "10")),
- Double.parseDouble(configs.getOrDefault("slope", "0")),
- Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
- break;
-
- case "dualband":
- series = getDualBandSeries(n,
- Double.parseDouble(configs.getOrDefault("lowBandValue", "10")),
- Double.parseDouble(configs.getOrDefault("lowBandDeviationPercentage", "0")),
- Integer.parseInt(configs.getOrDefault("lowBandPeriodSize", "0")),
- Double.parseDouble(configs.getOrDefault("highBandValue", "10")),
- Double.parseDouble(configs.getOrDefault("highBandDeviationPercentage", "0")),
- Integer.parseInt(configs.getOrDefault("highBandPeriodSize", "0")));
- break;
-
- case "step":
- series = getStepFunctionSeries(n,
- Double.parseDouble(configs.getOrDefault("startValue", "10")),
- Double.parseDouble(configs.getOrDefault("steadyValueDeviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("steadyPeriodSlope", "0")),
- Integer.parseInt(configs.getOrDefault("steadyPeriodMinSize", "0")),
- Integer.parseInt(configs.getOrDefault("steadyPeriodMaxSize", "0")),
- Double.parseDouble(configs.getOrDefault("stepChangePercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("upwardStep", "true")));
- break;
-
- case "turbulence":
- series = getSteadySeriesWithTurbulentPeriod(n,
- Double.parseDouble(configs.getOrDefault("steadyStateValue", "10")),
- Double.parseDouble(configs.getOrDefault("steadyStateDeviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationHigherPercentage", "10")),
- Integer.parseInt(configs.getOrDefault("turbulentPeriodLength", "0")),
- Integer.parseInt(configs.getOrDefault("turbulentStatePosition", "0")));
- break;
-
- default:
- series = createNormalSeries(n,
- 0,
- 1,
- 0,
- 0,
- 0,
- true);
- }
- return series;
- }
-
- public static AbstractMetricSeries generateSeries(String type, Map<String, String> configs) {
-
- AbstractMetricSeries series;
- switch (type) {
-
- case "normal":
- series = new NormalMetricSeries(Double.parseDouble(configs.getOrDefault("mean", "0")),
- Double.parseDouble(configs.getOrDefault("sd", "1")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesLower", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesHigher", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outlierOnRightEnd", "true")));
- break;
-
- case "uniform":
- series = new UniformMetricSeries(
- Double.parseDouble(configs.getOrDefault("value", "10")),
- Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
- break;
-
- case "monotonic":
- series = new MonotonicMetricSeries(
- Double.parseDouble(configs.getOrDefault("startValue", "10")),
- Double.parseDouble(configs.getOrDefault("slope", "0")),
- Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
- break;
-
- case "dualband":
- series = new DualBandMetricSeries(
- Double.parseDouble(configs.getOrDefault("lowBandValue", "10")),
- Double.parseDouble(configs.getOrDefault("lowBandDeviationPercentage", "0")),
- Integer.parseInt(configs.getOrDefault("lowBandPeriodSize", "0")),
- Double.parseDouble(configs.getOrDefault("highBandValue", "10")),
- Double.parseDouble(configs.getOrDefault("highBandDeviationPercentage", "0")),
- Integer.parseInt(configs.getOrDefault("highBandPeriodSize", "0")));
- break;
-
- case "step":
- series = new StepFunctionMetricSeries(
- Double.parseDouble(configs.getOrDefault("startValue", "10")),
- Double.parseDouble(configs.getOrDefault("steadyValueDeviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("steadyPeriodSlope", "0")),
- Integer.parseInt(configs.getOrDefault("steadyPeriodMinSize", "0")),
- Integer.parseInt(configs.getOrDefault("steadyPeriodMaxSize", "0")),
- Double.parseDouble(configs.getOrDefault("stepChangePercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("upwardStep", "true")));
- break;
-
- case "turbulence":
- series = new SteadyWithTurbulenceMetricSeries(
- Integer.parseInt(configs.getOrDefault("approxSeriesLength", "100")),
- Double.parseDouble(configs.getOrDefault("steadyStateValue", "10")),
- Double.parseDouble(configs.getOrDefault("steadyStateDeviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationHigherPercentage", "10")),
- Integer.parseInt(configs.getOrDefault("turbulentPeriodLength", "0")),
- Integer.parseInt(configs.getOrDefault("turbulentStatePosition", "0")));
- break;
-
- default:
- series = new NormalMetricSeries(0,
- 1,
- 0,
- 0,
- 0,
- true);
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java
deleted file mode 100644
index a883d08..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class MonotonicMetricSeries implements AbstractMetricSeries {
-
- double startValue = 0.0;
- double slope = 0.5;
- double deviationPercentage = 0.0;
- double outlierProbability = 0.0;
- double outlierDeviationLowerPercentage = 0.0;
- double outlierDeviationHigherPercentage = 0.0;
- boolean outliersAboveValue = true;
-
- Random random = new Random();
- double nonOutlierProbability;
-
- // y = mx + c
- double y;
- double m;
- double x;
- double c;
-
- public MonotonicMetricSeries(double startValue,
- double slope,
- double deviationPercentage,
- double outlierProbability,
- double outlierDeviationLowerPercentage,
- double outlierDeviationHigherPercentage,
- boolean outliersAboveValue) {
- this.startValue = startValue;
- this.slope = slope;
- this.deviationPercentage = deviationPercentage;
- this.outlierProbability = outlierProbability;
- this.outlierDeviationLowerPercentage = outlierDeviationLowerPercentage;
- this.outlierDeviationHigherPercentage = outlierDeviationHigherPercentage;
- this.outliersAboveValue = outliersAboveValue;
- init();
- }
-
- private void init() {
- y = startValue;
- m = slope;
- x = 1;
- c = y - (m * x);
- nonOutlierProbability = 1.0 - outlierProbability;
- }
-
- @Override
- public double nextValue() {
-
- double value;
- double probability = random.nextDouble();
-
- y = m * x + c;
- if (probability <= nonOutlierProbability) {
- double valueDeviationLowerLimit = y - deviationPercentage * y;
- double valueDeviationHigherLimit = y + deviationPercentage * y;
- value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
- } else {
- if (outliersAboveValue) {
- double outlierLowerLimit = y + outlierDeviationLowerPercentage * y;
- double outlierUpperLimit = y + outlierDeviationHigherPercentage * y;
- value = outlierLowerLimit + (outlierUpperLimit - outlierLowerLimit) * random.nextDouble();
- } else {
- double outlierLowerLimit = y - outlierDeviationLowerPercentage * y;
- double outlierUpperLimit = y - outlierDeviationHigherPercentage * y;
- value = outlierUpperLimit + (outlierLowerLimit - outlierUpperLimit) * random.nextDouble();
- }
- }
- x++;
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
- double[] series = new double[n];
- for (int i = 0; i < n; i++) {
- series[i] = nextValue();
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java
deleted file mode 100644
index cc83d2c..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class NormalMetricSeries implements AbstractMetricSeries {
-
- double mean = 0.0;
- double sd = 1.0;
- double outlierProbability = 0.0;
- double outlierDeviationSDTimesLower = 0.0;
- double outlierDeviationSDTimesHigher = 0.0;
- boolean outlierOnRightEnd = true;
-
- Random random = new Random();
- double nonOutlierProbability;
-
-
- public NormalMetricSeries(double mean,
- double sd,
- double outlierProbability,
- double outlierDeviationSDTimesLower,
- double outlierDeviationSDTimesHigher,
- boolean outlierOnRightEnd) {
- this.mean = mean;
- this.sd = sd;
- this.outlierProbability = outlierProbability;
- this.outlierDeviationSDTimesLower = outlierDeviationSDTimesLower;
- this.outlierDeviationSDTimesHigher = outlierDeviationSDTimesHigher;
- this.outlierOnRightEnd = outlierOnRightEnd;
- init();
- }
-
- private void init() {
- nonOutlierProbability = 1.0 - outlierProbability;
- }
-
- @Override
- public double nextValue() {
-
- double value;
- double probability = random.nextDouble();
-
- if (probability <= nonOutlierProbability) {
- value = random.nextGaussian() * sd + mean;
- } else {
- if (outlierOnRightEnd) {
- value = mean + (outlierDeviationSDTimesLower + (outlierDeviationSDTimesHigher - outlierDeviationSDTimesLower) * random.nextDouble()) * sd;
- } else {
- value = mean - (outlierDeviationSDTimesLower + (outlierDeviationSDTimesHigher - outlierDeviationSDTimesLower) * random.nextDouble()) * sd;
- }
- }
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
- double[] series = new double[n];
- for (int i = 0; i < n; i++) {
- series[i] = nextValue();
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java
deleted file mode 100644
index c4ed3ba..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class SteadyWithTurbulenceMetricSeries implements AbstractMetricSeries {
-
- double steadyStateValue = 0.0;
- double steadyStateDeviationPercentage = 0.0;
- double turbulentPeriodDeviationLowerPercentage = 0.3;
- double turbulentPeriodDeviationHigherPercentage = 0.5;
- int turbulentPeriodLength = 5;
- int turbulentStatePosition = 1;
- int approximateSeriesLength = 10;
-
- Random random = new Random();
- double valueDeviationLowerLimit;
- double valueDeviationHigherLimit;
- double tPeriodLowerLimit;
- double tPeriodUpperLimit;
- int tPeriodStartIndex = 0;
- int index = 0;
-
- public SteadyWithTurbulenceMetricSeries(int approximateSeriesLength,
- double steadyStateValue,
- double steadyStateDeviationPercentage,
- double turbulentPeriodDeviationLowerPercentage,
- double turbulentPeriodDeviationHigherPercentage,
- int turbulentPeriodLength,
- int turbulentStatePosition) {
- this.approximateSeriesLength = approximateSeriesLength;
- this.steadyStateValue = steadyStateValue;
- this.steadyStateDeviationPercentage = steadyStateDeviationPercentage;
- this.turbulentPeriodDeviationLowerPercentage = turbulentPeriodDeviationLowerPercentage;
- this.turbulentPeriodDeviationHigherPercentage = turbulentPeriodDeviationHigherPercentage;
- this.turbulentPeriodLength = turbulentPeriodLength;
- this.turbulentStatePosition = turbulentStatePosition;
- init();
- }
-
- private void init() {
-
- if (turbulentStatePosition == 1) {
- tPeriodStartIndex = (int) (0.25 * approximateSeriesLength + (0.25 * approximateSeriesLength * random.nextDouble()));
- } else if (turbulentStatePosition == 2) {
- tPeriodStartIndex = approximateSeriesLength - turbulentPeriodLength;
- }
-
- valueDeviationLowerLimit = steadyStateValue - steadyStateDeviationPercentage * steadyStateValue;
- valueDeviationHigherLimit = steadyStateValue + steadyStateDeviationPercentage * steadyStateValue;
-
- tPeriodLowerLimit = steadyStateValue + turbulentPeriodDeviationLowerPercentage * steadyStateValue;
- tPeriodUpperLimit = steadyStateValue + turbulentPeriodDeviationHigherPercentage * steadyStateValue;
- }
-
- @Override
- public double nextValue() {
-
- double value;
-
- if (index >= tPeriodStartIndex && index <= (tPeriodStartIndex + turbulentPeriodLength)) {
- value = tPeriodLowerLimit + (tPeriodUpperLimit - tPeriodLowerLimit) * random.nextDouble();
- } else {
- value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
- }
- index++;
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
-
- double[] series = new double[n];
- int turbulentPeriodStartIndex = 0;
-
- if (turbulentStatePosition == 1) {
- turbulentPeriodStartIndex = (int) (0.25 * n + (0.25 * n * random.nextDouble()));
- } else if (turbulentStatePosition == 2) {
- turbulentPeriodStartIndex = n - turbulentPeriodLength;
- }
-
- double valueDevLowerLimit = steadyStateValue - steadyStateDeviationPercentage * steadyStateValue;
- double valueDevHigherLimit = steadyStateValue + steadyStateDeviationPercentage * steadyStateValue;
-
- double turbulentPeriodLowerLimit = steadyStateValue + turbulentPeriodDeviationLowerPercentage * steadyStateValue;
- double turbulentPeriodUpperLimit = steadyStateValue + turbulentPeriodDeviationHigherPercentage * steadyStateValue;
-
- for (int i = 0; i < n; i++) {
- if (i >= turbulentPeriodStartIndex && i < (turbulentPeriodStartIndex + turbulentPeriodLength)) {
- series[i] = turbulentPeriodLowerLimit + (turbulentPeriodUpperLimit - turbulentPeriodLowerLimit) * random.nextDouble();
- } else {
- series[i] = valueDevLowerLimit + (valueDevHigherLimit - valueDevLowerLimit) * random.nextDouble();
- }
- }
-
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java
deleted file mode 100644
index d5beb48..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class StepFunctionMetricSeries implements AbstractMetricSeries {
-
- double startValue = 0.0;
- double steadyValueDeviationPercentage = 0.0;
- double steadyPeriodSlope = 0.5;
- int steadyPeriodMinSize = 10;
- int steadyPeriodMaxSize = 20;
- double stepChangePercentage = 0.0;
- boolean upwardStep = true;
-
- Random random = new Random();
-
- // y = mx + c
- double y;
- double m;
- double x;
- double c;
- int currentStepSize;
- int currentIndex;
-
- public StepFunctionMetricSeries(double startValue,
- double steadyValueDeviationPercentage,
- double steadyPeriodSlope,
- int steadyPeriodMinSize,
- int steadyPeriodMaxSize,
- double stepChangePercentage,
- boolean upwardStep) {
- this.startValue = startValue;
- this.steadyValueDeviationPercentage = steadyValueDeviationPercentage;
- this.steadyPeriodSlope = steadyPeriodSlope;
- this.steadyPeriodMinSize = steadyPeriodMinSize;
- this.steadyPeriodMaxSize = steadyPeriodMaxSize;
- this.stepChangePercentage = stepChangePercentage;
- this.upwardStep = upwardStep;
- init();
- }
-
- private void init() {
- y = startValue;
- m = steadyPeriodSlope;
- x = 1;
- c = y - (m * x);
-
- currentStepSize = (int) (steadyPeriodMinSize + (steadyPeriodMaxSize - steadyPeriodMinSize) * random.nextDouble());
- currentIndex = 0;
- }
-
- @Override
- public double nextValue() {
-
- double value = 0.0;
-
- if (currentIndex < currentStepSize) {
- y = m * x + c;
- double valueDeviationLowerLimit = y - steadyValueDeviationPercentage * y;
- double valueDeviationHigherLimit = y + steadyValueDeviationPercentage * y;
- value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
- x++;
- currentIndex++;
- }
-
- if (currentIndex == currentStepSize) {
- currentIndex = 0;
- currentStepSize = (int) (steadyPeriodMinSize + (steadyPeriodMaxSize - steadyPeriodMinSize) * random.nextDouble());
- if (upwardStep) {
- y = y + stepChangePercentage * y;
- } else {
- y = y - stepChangePercentage * y;
- }
- x = 1;
- c = y - (m * x);
- }
-
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
- double[] series = new double[n];
- for (int i = 0; i < n; i++) {
- series[i] = nextValue();
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java
deleted file mode 100644
index a2b0eea..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class UniformMetricSeries implements AbstractMetricSeries {
-
- double value = 0.0;
- double deviationPercentage = 0.0;
- double outlierProbability = 0.0;
- double outlierDeviationLowerPercentage = 0.0;
- double outlierDeviationHigherPercentage = 0.0;
- boolean outliersAboveValue= true;
-
- Random random = new Random();
- double valueDeviationLowerLimit;
- double valueDeviationHigherLimit;
- double outlierLeftLowerLimit;
- double outlierLeftHigherLimit;
- double outlierRightLowerLimit;
- double outlierRightUpperLimit;
- double nonOutlierProbability;
-
-
- public UniformMetricSeries(double value,
- double deviationPercentage,
- double outlierProbability,
- double outlierDeviationLowerPercentage,
- double outlierDeviationHigherPercentage,
- boolean outliersAboveValue) {
- this.value = value;
- this.deviationPercentage = deviationPercentage;
- this.outlierProbability = outlierProbability;
- this.outlierDeviationLowerPercentage = outlierDeviationLowerPercentage;
- this.outlierDeviationHigherPercentage = outlierDeviationHigherPercentage;
- this.outliersAboveValue = outliersAboveValue;
- init();
- }
-
- private void init() {
- valueDeviationLowerLimit = value - deviationPercentage * value;
- valueDeviationHigherLimit = value + deviationPercentage * value;
-
- outlierLeftLowerLimit = value - outlierDeviationHigherPercentage * value;
- outlierLeftHigherLimit = value - outlierDeviationLowerPercentage * value;
- outlierRightLowerLimit = value + outlierDeviationLowerPercentage * value;
- outlierRightUpperLimit = value + outlierDeviationHigherPercentage * value;
-
- nonOutlierProbability = 1.0 - outlierProbability;
- }
-
- @Override
- public double nextValue() {
-
- double value;
- double probability = random.nextDouble();
-
- if (probability <= nonOutlierProbability) {
- value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
- } else {
- if (!outliersAboveValue) {
- value = outlierLeftLowerLimit + (outlierLeftHigherLimit - outlierLeftLowerLimit) * random.nextDouble();
- } else {
- value = outlierRightLowerLimit + (outlierRightUpperLimit - outlierRightLowerLimit) * random.nextDouble();
- }
- }
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
- double[] series = new double[n];
- for (int i = 0; i < n; i++) {
- series[i] = nextValue();
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R
deleted file mode 100644
index 0b66095..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R
+++ /dev/null
@@ -1,96 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# EMA <- w * EMA + (1 - w) * x
-# EMS <- sqrt( w * EMS^2 + (1 - w) * (x - EMA)^2 )
-# Alarm = abs(x - EMA) > n * EMS
-
-ema_global <- function(train_data, test_data, w, n) {
-
-# res <- get_data(url)
-# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
-# names(data) <- c("TS", res$metrics[[1]]$metricname)
-# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2]
-# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
-
- anomalies <- data.frame()
- ema <- 0
- ems <- 0
-
- #Train Step
- for (x in train_data) {
- ema <- w*ema + (1-w)*x
- ems <- sqrt(w* ems^2 + (1 - w)*(x - ema)^2)
- }
-
- for ( i in 1:length(test_data[,1])) {
- x <- test_data[i,2]
- if (abs(x - ema) > n*ems) {
- anomaly <- c(as.numeric(test_data[i,1]), x)
- # print (anomaly)
- anomalies <- rbind(anomalies, anomaly)
- }
- ema <- w*ema + (1-w)*x
- ems <- sqrt(w* ems^2 + (1 - w)*(x - ema)^2)
- }
-
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS", "Value")
- }
- return (anomalies)
-}
-
-ema_daily <- function(train_data, test_data, w, n) {
-
-# res <- get_data(url)
-# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
-# names(data) <- c("TS", res$metrics[[1]]$metricname)
-# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), ]
-# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
-
- anomalies <- data.frame()
- ema <- vector("numeric", 7)
- ems <- vector("numeric", 7)
-
- #Train Step
- for ( i in 1:length(train_data[,1])) {
- x <- train_data[i,2]
- time <- as.POSIXlt(as.numeric(train_data[i,1])/1000, origin = "1970-01-01" ,tz = "GMT")
- index <- time$wday
- ema[index] <- w*ema[index] + (1-w)*x
- ems[index] <- sqrt(w* ems[index]^2 + (1 - w)*(x - ema[index])^2)
- }
-
- for ( i in 1:length(test_data[,1])) {
- x <- test_data[i,2]
- time <- as.POSIXlt(as.numeric(test_data[i,1])/1000, origin = "1970-01-01" ,tz = "GMT")
- index <- time$wday
-
- if (abs(x - ema[index+1]) > n*ems[index+1]) {
- anomaly <- c(as.numeric(test_data[i,1]), x)
- # print (anomaly)
- anomalies <- rbind(anomalies, anomaly)
- }
- ema[index+1] <- w*ema[index+1] + (1-w)*x
- ems[index+1] <- sqrt(w* ems[index+1]^2 + (1 - w)*(x - ema[index+1])^2)
- }
-
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS", "Value")
- }
- return(anomalies)
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r
deleted file mode 100644
index bca3366..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r
+++ /dev/null
@@ -1,67 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-hsdev_daily <- function(train_data, test_data, n, num_historic_periods, interval, period) {
-
- #res <- get_data(url)
- #data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
- #names(data) <- c("TS", res$metrics[[1]]$metricname)
- anomalies <- data.frame()
-
- granularity <- train_data[2,1] - train_data[1,1]
- test_start <- test_data[1,1]
- test_end <- test_data[length(test_data[1,]),1]
- train_start <- test_start - num_historic_periods*period
- # round to start of day
- train_start <- train_start - (train_start %% interval)
-
- time <- as.POSIXlt(as.numeric(test_data[1,1])/1000, origin = "1970-01-01" ,tz = "GMT")
- test_data_day <- time$wday
-
- h_data <- c()
- for ( i in length(train_data[,1]):1) {
- ts <- train_data[i,1]
- if ( ts < train_start) {
- break
- }
- time <- as.POSIXlt(as.numeric(ts)/1000, origin = "1970-01-01" ,tz = "GMT")
- if (time$wday == test_data_day) {
- x <- train_data[i,2]
- h_data <- c(h_data, x)
- }
- }
-
- if (length(h_data) < 2*length(test_data[,1])) {
- cat ("\nNot enough training data")
- return (anomalies)
- }
-
- past_median <- median(h_data)
- past_sd <- sd(h_data)
- curr_median <- median(test_data[,2])
-
- if (abs(curr_median - past_median) > n * past_sd) {
- anomaly <- c(test_start, test_end, curr_median, past_median, past_sd)
- anomalies <- rbind(anomalies, anomaly)
- }
-
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS Start", "TS End", "Current Median", "Past Median", "Past SD")
- }
-
- return (anomalies)
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R
deleted file mode 100644
index 8956400..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ams_iforest <- function(url, train_start, train_end, test_start, test_end, threshold_score) {
-
- res <- get_data(url)
- num_metrics <- length(res$metrics)
- anomalies <- data.frame()
-
- metricname <- res$metrics[[1]]$metricname
- data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
- names(data) <- c("TS", res$metrics[[1]]$metricname)
-
- for (i in 2:num_metrics) {
- metricname <- res$metrics[[i]]$metricname
- df <- data.frame(as.numeric(names(res$metrics[[i]]$metrics)), as.numeric(res$metrics[[i]]$metrics))
- names(df) <- c("TS", res$metrics[[i]]$metricname)
- data <- merge(data, df)
- }
-
- algo_data <- data[ which(df$TS >= train_start & df$TS <= train_end) , ][c(1:num_metrics+1)]
- iForest <- IsolationTrees(algo_data)
- test_data <- data[ which(df$TS >= test_start & df$TS <= test_end) , ]
-
- if_res <- AnomalyScore(test_data[c(1:num_metrics+1)], iForest)
- for (i in 1:length(if_res$outF)) {
- index <- test_start+i-1
- if (if_res$outF[i] > threshold_score) {
- anomaly <- c(test_data[i,1], if_res$outF[i], if_res$pathLength[i])
- anomalies <- rbind(anomalies, anomaly)
- }
- }
-
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS", "Anomaly Score", "Path length")
- }
- return (anomalies)
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r
deleted file mode 100644
index f22bc15..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ams_ks <- function(train_data, test_data, p_value) {
-
-# res <- get_data(url)
-# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
-# names(data) <- c("TS", res$metrics[[1]]$metricname)
-# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2]
-# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), 2]
-
- anomalies <- data.frame()
- res <- ks.test(train_data[,2], test_data[,2])
-
- if (res[2] < p_value) {
- anomaly <- c(test_data[1,1], test_data[length(test_data),1], res[1], res[2])
- anomalies <- rbind(anomalies, anomaly)
- }
-
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS Start", "TS End", "D", "p-value")
- }
- return (anomalies)
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R
deleted file mode 100644
index 7650356..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R
+++ /dev/null
@@ -1,85 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-tukeys_anomalies <- data.frame()
-ema_global_anomalies <- data.frame()
-ema_daily_anomalies <- data.frame()
-ks_anomalies <- data.frame()
-hsdev_anomalies <- data.frame()
-
-init <- function() {
- tukeys_anomalies <- data.frame()
- ema_global_anomalies <- data.frame()
- ema_daily_anomalies <- data.frame()
- ks_anomalies <- data.frame()
- hsdev_anomalies <- data.frame()
-}
-
-test_methods <- function(data) {
-
- init()
- #res <- get_data(url)
- #data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
- #names(data) <- c("TS", res$metrics[[1]]$metricname)
-
- limit <- data[length(data[,1]),1]
- step <- data[2,1] - data[1,1]
-
- train_start <- data[1,1]
- train_end <- get_next_day_boundary(train_start, step, limit)
- test_start <- train_end + step
- test_end <- get_next_day_boundary(test_start, step, limit)
- i <- 1
- day <- 24*60*60*1000
-
- while (test_start < limit) {
-
- print (i)
- i <- i + 1
- train_data <- data[which(data$TS >= train_start & data$TS <= train_end),]
- test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
-
- #tukeys_anomalies <<- rbind(tukeys_anomalies, ams_tukeys(train_data, test_data, 3))
- #ema_global_anomalies <<- rbind(ema_global_anomalies, ema_global(train_data, test_data, 0.9, 3))
- #ema_daily_anomalies <<- rbind(ema_daily_anomalies, ema_daily(train_data, test_data, 0.9, 3))
- #ks_anomalies <<- rbind(ks_anomalies, ams_ks(train_data, test_data, 0.05))
- hsdev_train_data <- data[which(data$TS < test_start),]
- hsdev_anomalies <<- rbind(hsdev_anomalies, hsdev_daily(hsdev_train_data, test_data, 3, 3, day, 7*day))
-
- train_start <- test_start
- train_end <- get_next_day_boundary(train_start, step, limit)
- test_start <- train_end + step
- test_end <- get_next_day_boundary(test_start, step, limit)
- }
- return (hsdev_anomalies)
-}
-
-get_next_day_boundary <- function(start, step, limit) {
-
- if (start > limit) {
- return (-1)
- }
-
- while (start <= limit) {
- if (((start %% (24*60*60*1000)) - 28800000) == 0) {
- return (start)
- }
- start <- start + step
- }
- return (start)
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r
deleted file mode 100644
index 0312226..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r
+++ /dev/null
@@ -1,51 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-ams_tukeys <- function(train_data, test_data, n) {
-
-# res <- get_data(url)
-# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
-# names(data) <- c("TS", res$metrics[[1]]$metricname)
-# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2]
-# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
-
- anomalies <- data.frame()
- quantiles <- quantile(train_data[,2])
- iqr <- quantiles[4] - quantiles[2]
- niqr <- 0
-
- for ( i in 1:length(test_data[,1])) {
- x <- test_data[i,2]
- lb <- quantiles[2] - n*iqr
- ub <- quantiles[4] + n*iqr
- if ( (x < lb) || (x > ub) ) {
- if (iqr != 0) {
- if (x < lb) {
- niqr <- (quantiles[2] - x) / iqr
- } else {
- niqr <- (x - quantiles[4]) / iqr
- }
- }
- anomaly <- c(test_data[i,1], x, niqr)
- anomalies <- rbind(anomalies, anomaly)
- }
- }
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS", "Value", "niqr")
- }
- return (anomalies)
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties
deleted file mode 100644
index ab106c4..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-appIds=HOST
-
-collectorHost=localhost
-collectorPort=6188
-collectorProtocol=http
-
-zkQuorum=localhost:2181
-
-ambariServerHost=localhost
-clusterName=c1
-
-emaW=0.8
-emaN=3
-tukeysN=3
-pointInTimeTestInterval=300000
-pointInTimeTrainInterval=900000
-
-ksTestInterval=600000
-ksTrainInterval=600000
-hsdevNhp=3
-hsdevInterval=1800000;
-
-skipMetricPatterns=sdisk*,cpu_sintr*,proc*,disk*,boottime
-hosts=avijayan-ad-1.openstacklocal
\ No newline at end of file