You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2017/09/27 22:03:07 UTC
[1/6] ambari git commit: AMBARI-22077 : Create maven module and
package structure for the anomaly detection engine. (Commit 2) (avijayan)
Repository: ambari
Updated Branches:
refs/heads/branch-3.0-ams e33b54557 -> 4613b471e
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/scala/org/apache/ambari/metrics/spark/MetricAnomalyDetector.scala
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/scala/org/apache/ambari/metrics/spark/MetricAnomalyDetector.scala b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/scala/org/apache/ambari/metrics/spark/MetricAnomalyDetector.scala
deleted file mode 100644
index 324058b..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/scala/org/apache/ambari/metrics/spark/MetricAnomalyDetector.scala
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.spark
-
-
-import java.io.{FileInputStream, IOException, InputStream}
-import java.util
-import java.util.Properties
-import java.util.logging.LogManager
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import org.apache.ambari.metrics.alertservice.prototype.core.MetricsCollectorInterface
-import org.apache.spark.SparkConf
-import org.apache.spark.streaming._
-import org.apache.spark.streaming.kafka._
-import org.apache.ambari.metrics.alertservice.prototype.methods.{AnomalyDetectionTechnique, MetricAnomaly}
-import org.apache.ambari.metrics.alertservice.prototype.methods.ema.{EmaModelLoader, EmaTechnique}
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics
-import org.apache.log4j.Logger
-import org.apache.spark.storage.StorageLevel
-
-object MetricAnomalyDetector {
-
- /*
- Load current EMA model
- Filter step - Check if anomaly
- Collect / Write to AMS / Print.
- */
-
-// var brokers = "avijayan-ams-1.openstacklocal:2181,avijayan-ams-2.openstacklocal:2181,avijayan-ams-3.openstacklocal:2181"
-// var groupId = "ambari-metrics-group"
-// var topicName = "ambari-metrics-topic"
-// var numThreads = 1
-// val anomalyDetectionModels: Array[AnomalyDetectionTechnique] = Array[AnomalyDetectionTechnique]()
-//
-// def readProperties(propertiesFile: String): Properties = try {
-// val properties = new Properties
-// var inputStream = ClassLoader.getSystemResourceAsStream(propertiesFile)
-// if (inputStream == null) inputStream = new FileInputStream(propertiesFile)
-// properties.load(inputStream)
-// properties
-// } catch {
-// case ioEx: IOException =>
-// null
-// }
-//
-// def main(args: Array[String]): Unit = {
-//
-// @transient
-// lazy val log = org.apache.log4j.LogManager.getLogger("MetricAnomalyDetectorLogger")
-//
-// if (args.length < 1) {
-// System.err.println("Usage: MetricSparkConsumer <input-config-file>")
-// System.exit(1)
-// }
-//
-// //Read properties
-// val properties = readProperties(propertiesFile = args(0))
-//
-// //Load EMA parameters - w, n
-// val emaW = properties.getProperty("emaW").toDouble
-// val emaN = properties.getProperty("emaN").toDouble
-//
-// //collector info
-// val collectorHost: String = properties.getProperty("collectorHost")
-// val collectorPort: String = properties.getProperty("collectorPort")
-// val collectorProtocol: String = properties.getProperty("collectorProtocol")
-// val anomalyMetricPublisher = new MetricsCollectorInterface(collectorHost, collectorProtocol, collectorPort)
-//
-// //Instantiate Kafka stream reader
-// val sparkConf = new SparkConf().setAppName("AmbariMetricsAnomalyDetector")
-// val streamingContext = new StreamingContext(sparkConf, Duration(10000))
-//
-// val topicsSet = topicName.toSet
-// val kafkaParams = Map[String, String]("metadata.broker.list" -> brokers)
-//// val stream = KafkaUtils.createDirectStream()
-//
-// val kafkaStream = KafkaUtils.createStream(streamingContext, zkQuorum, groupId, Map(topicName -> numThreads), StorageLevel.MEMORY_AND_DISK_SER_2)
-// kafkaStream.print()
-//
-// var timelineMetricsStream = kafkaStream.map( message => {
-// val mapper = new ObjectMapper
-// val metrics = mapper.readValue(message._2, classOf[TimelineMetrics])
-// metrics
-// })
-// timelineMetricsStream.print()
-//
-// var appMetricStream = timelineMetricsStream.map( timelineMetrics => {
-// (timelineMetrics.getMetrics.get(0).getAppId, timelineMetrics)
-// })
-// appMetricStream.print()
-//
-// var filteredAppMetricStream = appMetricStream.filter( appMetricTuple => {
-// appIds.contains(appMetricTuple._1)
-// } )
-// filteredAppMetricStream.print()
-//
-// filteredAppMetricStream.foreachRDD( rdd => {
-// rdd.foreach( appMetricTuple => {
-// val timelineMetrics = appMetricTuple._2
-// logger.info("Received Metric (1): " + timelineMetrics.getMetrics.get(0).getMetricName)
-// log.info("Received Metric (2): " + timelineMetrics.getMetrics.get(0).getMetricName)
-// for (timelineMetric <- timelineMetrics.getMetrics) {
-// var anomalies = emaModel.test(timelineMetric)
-// anomalyMetricPublisher.publish(anomalies)
-// }
-// })
-// })
-//
-// streamingContext.start()
-// streamingContext.awaitTermination()
-// }
- }
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/scala/org/apache/ambari/metrics/spark/SparkPhoenixReader.scala
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/scala/org/apache/ambari/metrics/spark/SparkPhoenixReader.scala b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/scala/org/apache/ambari/metrics/spark/SparkPhoenixReader.scala
deleted file mode 100644
index ccded6b..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/scala/org/apache/ambari/metrics/spark/SparkPhoenixReader.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.spark
-
-import org.apache.ambari.metrics.alertservice.prototype.methods.ema.EmaTechnique
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object SparkPhoenixReader {
-
- def main(args: Array[String]) {
-
- if (args.length < 6) {
- System.err.println("Usage: SparkPhoenixReader <metric_name> <appId> <hostname> <weight> <timessdev> <phoenixConnectionString> <model_dir>")
- System.exit(1)
- }
-
- var metricName = args(0)
- var appId = args(1)
- var hostname = args(2)
- var weight = args(3).toDouble
- var timessdev = args(4).toInt
- var phoenixConnectionString = args(5) //avijayan-ams-3.openstacklocal:61181:/ams-hbase-unsecure
- var modelDir = args(6)
-
- val conf = new SparkConf()
- conf.set("spark.app.name", "AMSAnomalyModelBuilder")
- //conf.set("spark.master", "spark://avijayan-ams-2.openstacklocal:7077")
-
- var sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- val currentTime = System.currentTimeMillis()
- val oneDayBack = currentTime - 24*60*60*1000
-
- val df = sqlContext.load("org.apache.phoenix.spark", Map("table" -> "METRIC_RECORD", "zkUrl" -> phoenixConnectionString))
- df.registerTempTable("METRIC_RECORD")
- val result = sqlContext.sql("SELECT METRIC_NAME, HOSTNAME, APP_ID, SERVER_TIME, METRIC_SUM, METRIC_COUNT FROM METRIC_RECORD " +
- "WHERE METRIC_NAME = '" + metricName + "' AND HOSTNAME = '" + hostname + "' AND APP_ID = '" + appId + "' AND SERVER_TIME < " + currentTime + " AND SERVER_TIME > " + oneDayBack)
-
- var metricValues = new java.util.TreeMap[java.lang.Long, java.lang.Double]
- result.collect().foreach(
- t => metricValues.put(t.getLong(3), t.getDouble(4) / t.getInt(5))
- )
-
- //val seriesName = result.head().getString(0)
- //val hostname = result.head().getString(1)
- //val appId = result.head().getString(2)
-
- val timelineMetric = new TimelineMetric()
- timelineMetric.setMetricName(metricName)
- timelineMetric.setAppId(appId)
- timelineMetric.setHostName(hostname)
- timelineMetric.setMetricValues(metricValues)
-
- var emaModel = new EmaTechnique(weight, timessdev)
- emaModel.test(timelineMetric)
- emaModel.save(sc, modelDir)
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestEmaTechnique.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestEmaTechnique.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestEmaTechnique.java
deleted file mode 100644
index a0b06e6..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestEmaTechnique.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype;
-
-import org.apache.ambari.metrics.alertservice.prototype.core.RFunctionInvoker;
-import org.apache.ambari.metrics.alertservice.prototype.methods.MetricAnomaly;
-import org.apache.ambari.metrics.alertservice.prototype.methods.ema.EmaTechnique;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.List;
-import java.util.TreeMap;
-
-import static org.apache.ambari.metrics.alertservice.prototype.TestRFunctionInvoker.getTS;
-
-public class TestEmaTechnique {
-
- private static double[] ts;
- private static String fullFilePath;
-
- @BeforeClass
- public static void init() throws URISyntaxException {
-
- Assume.assumeTrue(System.getenv("R_HOME") != null);
- ts = getTS(1000);
- URL url = ClassLoader.getSystemResource("R-scripts");
- fullFilePath = new File(url.toURI()).getAbsolutePath();
- RFunctionInvoker.setScriptsDir(fullFilePath);
- }
-
- @Test
- public void testEmaInitialization() {
-
- EmaTechnique ema = new EmaTechnique(0.5, 3);
- Assert.assertTrue(ema.getTrackedEmas().isEmpty());
- Assert.assertTrue(ema.getStartingWeight() == 0.5);
- Assert.assertTrue(ema.getStartTimesSdev() == 2);
- }
-
- @Test
- public void testEma() {
- EmaTechnique ema = new EmaTechnique(0.5, 3);
-
- long now = System.currentTimeMillis();
-
- TimelineMetric metric1 = new TimelineMetric();
- metric1.setMetricName("M1");
- metric1.setHostName("H1");
- metric1.setStartTime(now - 1000);
- metric1.setAppId("A1");
- metric1.setInstanceId(null);
- metric1.setType("Integer");
-
- //Train
- TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
- for (int i = 0; i < 50; i++) {
- double metric = 20000 + Math.random();
- metricValues.put(now - i * 100, metric);
- }
- metric1.setMetricValues(metricValues);
- List<MetricAnomaly> anomalyList = ema.test(metric1);
-// Assert.assertTrue(anomalyList.isEmpty());
-
- metricValues = new TreeMap<Long, Double>();
- for (int i = 0; i < 50; i++) {
- double metric = 20000 + Math.random();
- metricValues.put(now - i * 100, metric);
- }
- metric1.setMetricValues(metricValues);
- anomalyList = ema.test(metric1);
- Assert.assertTrue(!anomalyList.isEmpty());
- int l1 = anomalyList.size();
-
- Assert.assertTrue(ema.updateModel(metric1, false, 20));
- anomalyList = ema.test(metric1);
- int l2 = anomalyList.size();
- Assert.assertTrue(l2 < l1);
-
- Assert.assertTrue(ema.updateModel(metric1, true, 50));
- anomalyList = ema.test(metric1);
- int l3 = anomalyList.size();
- Assert.assertTrue(l3 > l2 && l3 > l1);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestRFunctionInvoker.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestRFunctionInvoker.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestRFunctionInvoker.java
deleted file mode 100644
index d98ef0c..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestRFunctionInvoker.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype;
-
-import org.apache.ambari.metrics.alertservice.prototype.common.ResultSet;
-import org.apache.ambari.metrics.alertservice.prototype.common.DataSeries;
-import org.apache.ambari.metrics.alertservice.prototype.core.RFunctionInvoker;
-import org.apache.ambari.metrics.alertservice.seriesgenerator.UniformMetricSeries;
-import org.apache.commons.lang.ArrayUtils;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-
-public class TestRFunctionInvoker {
-
- private static String metricName = "TestMetric";
- private static double[] ts;
- private static String fullFilePath;
-
- @BeforeClass
- public static void init() throws URISyntaxException {
-
- Assume.assumeTrue(System.getenv("R_HOME") != null);
- ts = getTS(1000);
- URL url = ClassLoader.getSystemResource("R-scripts");
- fullFilePath = new File(url.toURI()).getAbsolutePath();
- RFunctionInvoker.setScriptsDir(fullFilePath);
- }
-
- @Test
- public void testTukeys() throws URISyntaxException {
-
- double[] train_ts = ArrayUtils.subarray(ts, 0, 750);
- double[] train_x = getRandomData(750);
- DataSeries trainData = new DataSeries(metricName, train_ts, train_x);
-
- double[] test_ts = ArrayUtils.subarray(ts, 750, 1000);
- double[] test_x = getRandomData(250);
- test_x[50] = 5.5; //Anomaly
- DataSeries testData = new DataSeries(metricName, test_ts, test_x);
- Map<String, String> configs = new HashMap();
- configs.put("tukeys.n", "3");
-
- ResultSet rs = RFunctionInvoker.tukeys(trainData, testData, configs);
- Assert.assertEquals(rs.resultset.size(), 2);
- Assert.assertEquals(rs.resultset.get(1)[0], 5.5, 0.1);
-
- }
-
- public static void main(String[] args) throws URISyntaxException {
-
- String metricName = "TestMetric";
- double[] ts = getTS(1000);
- URL url = ClassLoader.getSystemResource("R-scripts");
- String fullFilePath = new File(url.toURI()).getAbsolutePath();
- RFunctionInvoker.setScriptsDir(fullFilePath);
-
- double[] train_ts = ArrayUtils.subarray(ts, 0, 750);
- double[] train_x = getRandomData(750);
- DataSeries trainData = new DataSeries(metricName, train_ts, train_x);
-
- double[] test_ts = ArrayUtils.subarray(ts, 750, 1000);
- double[] test_x = getRandomData(250);
- test_x[50] = 5.5; //Anomaly
- DataSeries testData = new DataSeries(metricName, test_ts, test_x);
- ResultSet rs;
-
- Map<String, String> configs = new HashMap();
-
- System.out.println("TUKEYS");
- configs.put("tukeys.n", "3");
- rs = RFunctionInvoker.tukeys(trainData, testData, configs);
- rs.print();
- System.out.println("--------------");
-
-// System.out.println("EMA Global");
-// configs.put("ema.n", "3");
-// configs.put("ema.w", "0.8");
-// rs = RFunctionInvoker.ema_global(trainData, testData, configs);
-// rs.print();
-// System.out.println("--------------");
-//
-// System.out.println("EMA Daily");
-// rs = RFunctionInvoker.ema_daily(trainData, testData, configs);
-// rs.print();
-// System.out.println("--------------");
-//
-// configs.put("ks.p_value", "0.00005");
-// System.out.println("KS Test");
-// rs = RFunctionInvoker.ksTest(trainData, testData, configs);
-// rs.print();
-// System.out.println("--------------");
-//
- ts = getTS(5000);
- train_ts = ArrayUtils.subarray(ts, 0, 4800);
- train_x = getRandomData(4800);
- trainData = new DataSeries(metricName, train_ts, train_x);
- test_ts = ArrayUtils.subarray(ts, 4800, 5000);
- test_x = getRandomData(200);
- for (int i = 0; i < 200; i++) {
- test_x[i] = test_x[i] * 5;
- }
- testData = new DataSeries(metricName, test_ts, test_x);
- configs.put("hsdev.n", "3");
- configs.put("hsdev.nhp", "3");
- configs.put("hsdev.interval", "86400000");
- configs.put("hsdev.period", "604800000");
- System.out.println("HSdev");
- rs = RFunctionInvoker.hsdev(trainData, testData, configs);
- rs.print();
- System.out.println("--------------");
-
- }
-
- static double[] getTS(int n) {
- long currentTime = System.currentTimeMillis();
- double[] ts = new double[n];
- currentTime = currentTime - (currentTime % (5 * 60 * 1000));
-
- for (int i = 0, j = n - 1; i < n; i++, j--) {
- ts[j] = currentTime;
- currentTime = currentTime - (5 * 60 * 1000);
- }
- return ts;
- }
-
- static double[] getRandomData(int n) {
-
- UniformMetricSeries metricSeries = new UniformMetricSeries(10, 0.1,0.05, 0.6, 0.8, true);
- return metricSeries.getSeries(n);
-
-// double[] metrics = new double[n];
-// Random random = new Random();
-// for (int i = 0; i < n; i++) {
-// metrics[i] = random.nextDouble();
-// }
-// return metrics;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestTukeys.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestTukeys.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestTukeys.java
deleted file mode 100644
index 86590bd..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/prototype/TestTukeys.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype;
-
-import org.apache.ambari.metrics.alertservice.prototype.core.MetricsCollectorInterface;
-import org.apache.ambari.metrics.alertservice.prototype.core.RFunctionInvoker;
-import org.apache.ambari.metrics.alertservice.prototype.methods.MetricAnomaly;
-import org.apache.ambari.metrics.alertservice.prototype.methods.ema.EmaTechnique;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.junit.Assume;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.util.List;
-import java.util.TreeMap;
-
-public class TestTukeys {
-
- @BeforeClass
- public static void init() throws URISyntaxException {
- Assume.assumeTrue(System.getenv("R_HOME") != null);
- }
-
- @Test
- public void testPointInTimeDetectionSystem() throws UnknownHostException, URISyntaxException {
-
- URL url = ClassLoader.getSystemResource("R-scripts");
- String fullFilePath = new File(url.toURI()).getAbsolutePath();
- RFunctionInvoker.setScriptsDir(fullFilePath);
-
- MetricsCollectorInterface metricsCollectorInterface = new MetricsCollectorInterface("avijayan-ams-1.openstacklocal","http", "6188");
-
- EmaTechnique ema = new EmaTechnique(0.5, 3);
- long now = System.currentTimeMillis();
-
- TimelineMetric metric1 = new TimelineMetric();
- metric1.setMetricName("mm9");
- metric1.setHostName(MetricsCollectorInterface.getDefaultLocalHostName());
- metric1.setStartTime(now);
- metric1.setAppId("aa9");
- metric1.setInstanceId(null);
- metric1.setType("Integer");
-
- //Train
- TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
-
- //2hr data.
- for (int i = 0; i < 120; i++) {
- double metric = 20000 + Math.random();
- metricValues.put(now - i * 60 * 1000, metric);
- }
- metric1.setMetricValues(metricValues);
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- timelineMetrics.addOrMergeTimelineMetric(metric1);
-
- metricsCollectorInterface.emitMetrics(timelineMetrics);
-
- List<MetricAnomaly> anomalyList = ema.test(metric1);
- metricsCollectorInterface.publish(anomalyList);
-//
-// PointInTimeADSystem pointInTimeADSystem = new PointInTimeADSystem(ema, metricsCollectorInterface, 3, 5*60*1000, 15*60*1000);
-// pointInTimeADSystem.runOnce();
-//
-// List<MetricAnomaly> anomalyList2 = ema.test(metric1);
-//
-// pointInTimeADSystem.runOnce();
-// List<MetricAnomaly> anomalyList3 = ema.test(metric1);
-//
-// pointInTimeADSystem.runOnce();
-// List<MetricAnomaly> anomalyList4 = ema.test(metric1);
-//
-// pointInTimeADSystem.runOnce();
-// List<MetricAnomaly> anomalyList5 = ema.test(metric1);
-//
-// pointInTimeADSystem.runOnce();
-// List<MetricAnomaly> anomalyList6 = ema.test(metric1);
-//
-// Assert.assertTrue(anomalyList6.size() < anomalyList.size());
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorTest.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorTest.java
deleted file mode 100644
index fe7dba9..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/test/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-public class MetricSeriesGeneratorTest {
-
- @Test
- public void testUniformSeries() {
-
- UniformMetricSeries metricSeries = new UniformMetricSeries(5, 0.2, 0, 0, 0, true);
- Assert.assertTrue(metricSeries.nextValue() <= 6 && metricSeries.nextValue() >= 4);
-
- double[] uniformSeries = MetricSeriesGeneratorFactory.createUniformSeries(50, 10, 0.2, 0.1, 0.4, 0.5, true);
- Assert.assertTrue(uniformSeries.length == 50);
-
- for (int i = 0; i < uniformSeries.length; i++) {
- double value = uniformSeries[i];
-
- if (value > 10 * 1.2) {
- Assert.assertTrue(value >= 10 * 1.4 && value <= 10 * 1.6);
- } else {
- Assert.assertTrue(value >= 10 * 0.8 && value <= 10 * 1.2);
- }
- }
- }
-
- @Test
- public void testNormalSeries() {
- NormalMetricSeries metricSeries = new NormalMetricSeries(0, 1, 0, 0, 0, true);
- Assert.assertTrue(metricSeries.nextValue() <= 3 && metricSeries.nextValue() >= -3);
- }
-
- @Test
- public void testMonotonicSeries() {
-
- MonotonicMetricSeries metricSeries = new MonotonicMetricSeries(0, 0.5, 0, 0, 0, 0, true);
- Assert.assertTrue(metricSeries.nextValue() == 0);
- Assert.assertTrue(metricSeries.nextValue() == 0.5);
-
- double[] incSeries = MetricSeriesGeneratorFactory.createMonotonicSeries(20, 0, 0.5, 0, 0, 0, 0, true);
- Assert.assertTrue(incSeries.length == 20);
- for (int i = 0; i < incSeries.length; i++) {
- Assert.assertTrue(incSeries[i] == i * 0.5);
- }
- }
-
- @Test
- public void testDualBandSeries() {
- double[] dualBandSeries = MetricSeriesGeneratorFactory.getDualBandSeries(30, 5, 0.2, 5, 15, 0.3, 4);
- Assert.assertTrue(dualBandSeries[0] >= 4 && dualBandSeries[0] <= 6);
- Assert.assertTrue(dualBandSeries[4] >= 4 && dualBandSeries[4] <= 6);
- Assert.assertTrue(dualBandSeries[5] >= 10.5 && dualBandSeries[5] <= 19.5);
- Assert.assertTrue(dualBandSeries[8] >= 10.5 && dualBandSeries[8] <= 19.5);
- Assert.assertTrue(dualBandSeries[9] >= 4 && dualBandSeries[9] <= 6);
- }
-
- @Test
- public void testStepSeries() {
- double[] stepSeries = MetricSeriesGeneratorFactory.getStepFunctionSeries(30, 10, 0, 0, 5, 5, 0.5, true);
-
- Assert.assertTrue(stepSeries[0] == 10);
- Assert.assertTrue(stepSeries[4] == 10);
-
- Assert.assertTrue(stepSeries[5] == 10*1.5);
- Assert.assertTrue(stepSeries[9] == 10*1.5);
-
- Assert.assertTrue(stepSeries[10] == 10*1.5*1.5);
- Assert.assertTrue(stepSeries[14] == 10*1.5*1.5);
- }
-
- @Test
- public void testSteadySeriesWithTurbulence() {
- double[] steadySeriesWithTurbulence = MetricSeriesGeneratorFactory.getSteadySeriesWithTurbulentPeriod(30, 5, 0, 1, 1, 5, 1);
-
- int count = 0;
- for (int i = 0; i < steadySeriesWithTurbulence.length; i++) {
- if (steadySeriesWithTurbulence[i] == 10) {
- count++;
- }
- }
- Assert.assertTrue(count == 5);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml
index a68ac63..fb2997a 100644
--- a/ambari-metrics/pom.xml
+++ b/ambari-metrics/pom.xml
@@ -34,7 +34,7 @@
<module>ambari-metrics-grafana</module>
<module>ambari-metrics-assembly</module>
<module>ambari-metrics-host-aggregator</module>
- <module>ambari-metrics-anomaly-detector</module>
+ <module>ambari-metrics-anomaly-detection-service</module>
</modules>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
[5/6] ambari git commit: AMBARI-22077 : Create maven module and
package structure for the anomaly detection engine. (Commit 2) (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/MetricAnomaly.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/MetricAnomaly.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/MetricAnomaly.java
new file mode 100644
index 0000000..251603b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/MetricAnomaly.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.methods;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+public class MetricAnomaly implements Serializable{
+
+ private String methodType;
+ private double anomalyScore;
+ private String metricKey;
+ private long timestamp;
+ private double metricValue;
+
+
+ public MetricAnomaly(String metricKey, long timestamp, double metricValue, String methodType, double anomalyScore) {
+ this.metricKey = metricKey;
+ this.timestamp = timestamp;
+ this.metricValue = metricValue;
+ this.methodType = methodType;
+ this.anomalyScore = anomalyScore;
+
+ }
+
+ public String getMethodType() {
+ return methodType;
+ }
+
+ public void setMethodType(String methodType) {
+ this.methodType = methodType;
+ }
+
+ public double getAnomalyScore() {
+ return anomalyScore;
+ }
+
+ public void setAnomalyScore(double anomalyScore) {
+ this.anomalyScore = anomalyScore;
+ }
+
+ public void setMetricKey(String metricKey) {
+ this.metricKey = metricKey;
+ }
+
+ public String getMetricKey() {
+ return metricKey;
+ }
+
+ public void setMetricName(String metricName) {
+ this.metricKey = metricName;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public double getMetricValue() {
+ return metricValue;
+ }
+
+ public void setMetricValue(double metricValue) {
+ this.metricValue = metricValue;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaModel.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaModel.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaModel.java
new file mode 100644
index 0000000..593028e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaModel.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.methods.ema;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+
+import static org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaTechnique.suppressAnomaliesTheshold;
+
+@XmlRootElement
+public class EmaModel implements Serializable {
+
+ private String metricName;
+ private String hostname;
+ private String appId;
+ private double ema;
+ private double ems;
+ private double weight;
+ private double timessdev;
+
+ private int ctr = 0;
+
+ private static final Log LOG = LogFactory.getLog(EmaModel.class);
+
+ public EmaModel(String name, String hostname, String appId, double weight, double timessdev) {
+ this.metricName = name;
+ this.hostname = hostname;
+ this.appId = appId;
+ this.weight = weight;
+ this.timessdev = timessdev;
+ this.ema = 0.0;
+ this.ems = 0.0;
+ }
+
+ public String getMetricName() {
+ return metricName;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ public double testAndUpdate(double metricValue) {
+
+ double anomalyScore = 0.0;
+ LOG.info("Before Update ->" + metricName + ":" + appId + ":" + hostname + " - " + "ema = " + ema + ", ems = " + ems + ", timessdev = " + timessdev);
+ update(metricValue);
+ if (ctr > suppressAnomaliesTheshold) {
+ anomalyScore = test(metricValue);
+ if (anomalyScore > 0.0) {
+ LOG.info("Anomaly ->" + metricName + ":" + appId + ":" + hostname + " - " + "ema = " + ema + ", ems = " + ems +
+ ", timessdev = " + timessdev + ", metricValue = " + metricValue);
+ } else {
+ LOG.info("Not an Anomaly ->" + metricName + ":" + appId + ":" + hostname + " - " + "ema = " + ema + ", ems = " + ems +
+ ", timessdev = " + timessdev + ", metricValue = " + metricValue);
+ }
+ } else {
+ ctr++;
+ if (ctr > suppressAnomaliesTheshold) {
+ LOG.info("Ema Model for " + metricName + ":" + appId + ":" + hostname + " is ready for testing data.");
+ }
+ }
+ return anomalyScore;
+ }
+
+ public void update(double metricValue) {
+ ema = weight * ema + (1 - weight) * metricValue;
+ ems = Math.sqrt(weight * Math.pow(ems, 2.0) + (1 - weight) * Math.pow(metricValue - ema, 2.0));
+ LOG.debug("In update : ema = " + ema + ", ems = " + ems);
+ }
+
+ public double test(double metricValue) {
+ LOG.debug("In test : ema = " + ema + ", ems = " + ems);
+ double diff = Math.abs(ema - metricValue) - (timessdev * ems);
+ LOG.debug("diff = " + diff);
+ if (diff > 0) {
+ return Math.abs((metricValue - ema) / ems); //Z score
+ } else {
+ return 0.0;
+ }
+ }
+
+ public void updateModel(boolean increaseSensitivity, double percent) {
+ LOG.info("Updating model for " + metricName + " with increaseSensitivity = " + increaseSensitivity + ", percent = " + percent);
+ double delta = percent / 100;
+ if (increaseSensitivity) {
+ delta = delta * -1;
+ }
+ this.timessdev = timessdev + delta * timessdev;
+ //this.weight = Math.min(1.0, weight + delta * weight);
+ LOG.info("New model parameters " + metricName + " : timessdev = " + timessdev + ", weight = " + weight);
+ }
+
+ public double getWeight() {
+ return weight;
+ }
+
+ public void setWeight(double weight) {
+ this.weight = weight;
+ }
+
+ public double getTimessdev() {
+ return timessdev;
+ }
+
+ public void setTimessdev(double timessdev) {
+ this.timessdev = timessdev;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaModelLoader.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaModelLoader.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaModelLoader.java
new file mode 100644
index 0000000..7623f27
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaModelLoader.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.methods.ema;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.spark.SparkContext;
+import org.apache.spark.mllib.util.Loader;
+
+public class EmaModelLoader implements Loader<EmaTechnique> {
+ private static final Log LOG = LogFactory.getLog(EmaModelLoader.class);
+
+ @Override
+ public EmaTechnique load(SparkContext sc, String path) {
+ return new EmaTechnique(0.5,3);
+// Gson gson = new Gson();
+// try {
+// String fileString = new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8);
+// return gson.fromJson(fileString, EmaTechnique.class);
+// } catch (IOException e) {
+// LOG.error(e);
+// }
+// return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaTechnique.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaTechnique.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaTechnique.java
new file mode 100644
index 0000000..7ec17d8
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/ema/EmaTechnique.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.methods.ema;
+
+import com.google.gson.Gson;
+import org.apache.ambari.metrics.adservice.prototype.methods.AnomalyDetectionTechnique;
+import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.spark.SparkContext;
+import org.apache.spark.mllib.util.Saveable;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.BufferedWriter;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@XmlRootElement
+public class EmaTechnique extends AnomalyDetectionTechnique implements Serializable, Saveable {
+
+ @XmlElement(name = "trackedEmas")
+ private Map<String, EmaModel> trackedEmas;
+ private static final Log LOG = LogFactory.getLog(EmaTechnique.class);
+
+ private double startingWeight = 0.5;
+ private double startTimesSdev = 3.0;
+ private String methodType = "ema";
+ public static int suppressAnomaliesTheshold = 100;
+
+ public EmaTechnique(double startingWeight, double startTimesSdev, int suppressAnomaliesTheshold) {
+ trackedEmas = new HashMap<>();
+ this.startingWeight = startingWeight;
+ this.startTimesSdev = startTimesSdev;
+ EmaTechnique.suppressAnomaliesTheshold = suppressAnomaliesTheshold;
+ LOG.info("New EmaTechnique......");
+ }
+
+ public EmaTechnique(double startingWeight, double startTimesSdev) {
+ trackedEmas = new HashMap<>();
+ this.startingWeight = startingWeight;
+ this.startTimesSdev = startTimesSdev;
+ LOG.info("New EmaTechnique......");
+ }
+
+ public List<MetricAnomaly> test(TimelineMetric metric) {
+ String metricName = metric.getMetricName();
+ String appId = metric.getAppId();
+ String hostname = metric.getHostName();
+ String key = metricName + ":" + appId + ":" + hostname;
+
+ EmaModel emaModel = trackedEmas.get(key);
+ if (emaModel == null) {
+ LOG.debug("EmaModel not present for " + key);
+ LOG.debug("Number of tracked Emas : " + trackedEmas.size());
+ emaModel = new EmaModel(metricName, hostname, appId, startingWeight, startTimesSdev);
+ trackedEmas.put(key, emaModel);
+ } else {
+ LOG.debug("EmaModel already present for " + key);
+ }
+
+ List<MetricAnomaly> anomalies = new ArrayList<>();
+
+ for (Long timestamp : metric.getMetricValues().keySet()) {
+ double metricValue = metric.getMetricValues().get(timestamp);
+ double anomalyScore = emaModel.testAndUpdate(metricValue);
+ if (anomalyScore > 0.0) {
+ LOG.info("Found anomaly for : " + key + ", anomalyScore = " + anomalyScore);
+ MetricAnomaly metricAnomaly = new MetricAnomaly(key, timestamp, metricValue, methodType, anomalyScore);
+ anomalies.add(metricAnomaly);
+ } else {
+ LOG.debug("Discarding non-anomaly for : " + key);
+ }
+ }
+ return anomalies;
+ }
+
+ public boolean updateModel(TimelineMetric timelineMetric, boolean increaseSensitivity, double percent) {
+ String metricName = timelineMetric.getMetricName();
+ String appId = timelineMetric.getAppId();
+ String hostname = timelineMetric.getHostName();
+ String key = metricName + "_" + appId + "_" + hostname;
+
+
+ EmaModel emaModel = trackedEmas.get(key);
+
+ if (emaModel == null) {
+ LOG.warn("EMA Model for " + key + " not found");
+ return false;
+ }
+ emaModel.updateModel(increaseSensitivity, percent);
+
+ return true;
+ }
+
+ @Override
+ public void save(SparkContext sc, String path) {
+ Gson gson = new Gson();
+ try {
+ String json = gson.toJson(this);
+ try (Writer writer = new BufferedWriter(new OutputStreamWriter(
+ new FileOutputStream(path), "utf-8"))) {
+ writer.write(json);
+ }
+ } catch (IOException e) {
+ LOG.error(e);
+ }
+ }
+
+ @Override
+ public String formatVersion() {
+ return "1.0";
+ }
+
+ public Map<String, EmaModel> getTrackedEmas() {
+ return trackedEmas;
+ }
+
+ public double getStartingWeight() {
+ return startingWeight;
+ }
+
+ public double getStartTimesSdev() {
+ return startTimesSdev;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/hsdev/HsdevTechnique.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/hsdev/HsdevTechnique.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/hsdev/HsdevTechnique.java
new file mode 100644
index 0000000..6facc99
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/hsdev/HsdevTechnique.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.methods.hsdev;
+
+import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
+import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import static org.apache.ambari.metrics.adservice.prototype.common.StatisticUtils.median;
+import static org.apache.ambari.metrics.adservice.prototype.common.StatisticUtils.sdev;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+public class HsdevTechnique implements Serializable {
+
+ private Map<String, Double> hsdevMap;
+ private String methodType = "hsdev";
+ private static final Log LOG = LogFactory.getLog(HsdevTechnique.class);
+
+ public HsdevTechnique() {
+ hsdevMap = new HashMap<>();
+ }
+
+ public MetricAnomaly runHsdevTest(String key, DataSeries trainData, DataSeries testData) {
+ int testLength = testData.values.length;
+ int trainLength = trainData.values.length;
+
+ if (trainLength < testLength) {
+ LOG.info("Not enough train data.");
+ return null;
+ }
+
+ if (!hsdevMap.containsKey(key)) {
+ hsdevMap.put(key, 3.0);
+ }
+
+ double n = hsdevMap.get(key);
+
+ double historicSd = sdev(trainData.values, false);
+ double historicMedian = median(trainData.values);
+ double currentMedian = median(testData.values);
+
+
+ if (historicSd > 0) {
+ double diff = Math.abs(currentMedian - historicMedian);
+ LOG.info("Found anomaly for metric : " + key + " in the period ending " + new Date((long)testData.ts[testLength - 1]));
+ LOG.info("Current median = " + currentMedian + ", Historic Median = " + historicMedian + ", HistoricSd = " + historicSd);
+
+ if (diff > n * historicSd) {
+ double zScore = diff / historicSd;
+ LOG.info("Z Score of current series : " + zScore);
+ return new MetricAnomaly(key,
+ (long) testData.ts[testLength - 1],
+ testData.values[testLength - 1],
+ methodType,
+ zScore);
+ }
+ }
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/kstest/KSTechnique.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/kstest/KSTechnique.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/kstest/KSTechnique.java
new file mode 100644
index 0000000..4727c6f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/kstest/KSTechnique.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.adservice.prototype.methods.kstest;
+
+import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
+import org.apache.ambari.metrics.adservice.prototype.common.ResultSet;
+import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
+import org.apache.ambari.metrics.adservice.prototype.core.RFunctionInvoker;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class KSTechnique implements Serializable {
+
+ private String methodType = "ks";
+ private Map<String, Double> pValueMap;
+ private static final Log LOG = LogFactory.getLog(KSTechnique.class);
+
+ public KSTechnique() {
+ pValueMap = new HashMap();
+ }
+
+ public MetricAnomaly runKsTest(String key, DataSeries trainData, DataSeries testData) {
+
+ int testLength = testData.values.length;
+ int trainLength = trainData.values.length;
+
+ if (trainLength < testLength) {
+ LOG.info("Not enough train data.");
+ return null;
+ }
+
+ if (!pValueMap.containsKey(key)) {
+ pValueMap.put(key, 0.05);
+ }
+ double pValue = pValueMap.get(key);
+
+ ResultSet result = RFunctionInvoker.ksTest(trainData, testData, Collections.singletonMap("ks.p_value", String.valueOf(pValue)));
+ if (result == null) {
+ LOG.error("Resultset is null when invoking KS R function...");
+ return null;
+ }
+
+ if (result.resultset.size() > 0) {
+
+ LOG.info("Is size 1 ? result size = " + result.resultset.get(0).length);
+ LOG.info("p_value = " + result.resultset.get(3)[0]);
+ double dValue = result.resultset.get(2)[0];
+
+ return new MetricAnomaly(key,
+ (long) testData.ts[testLength - 1],
+ testData.values[testLength - 1],
+ methodType,
+ dValue);
+ }
+
+ return null;
+ }
+
+ public void updateModel(String metricKey, boolean increaseSensitivity, double percent) {
+
+ LOG.info("Updating KS model for " + metricKey + " with increaseSensitivity = " + increaseSensitivity + ", percent = " + percent);
+
+ if (!pValueMap.containsKey(metricKey)) {
+ LOG.error("Unknown metric key : " + metricKey);
+ LOG.info("pValueMap :" + pValueMap.toString());
+ return;
+ }
+
+ double delta = percent / 100;
+ if (!increaseSensitivity) {
+ delta = delta * -1;
+ }
+
+ double pValue = pValueMap.get(metricKey);
+ double newPValue = Math.min(1.0, pValue + delta * pValue);
+ pValueMap.put(metricKey, newPValue);
+ LOG.info("New pValue = " + newPValue);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/MetricAnomalyDetectorTestInput.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/MetricAnomalyDetectorTestInput.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/MetricAnomalyDetectorTestInput.java
new file mode 100644
index 0000000..9a002a1
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/MetricAnomalyDetectorTestInput.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.adservice.prototype.testing.utilities;
+
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.List;
+import java.util.Map;
+
+@XmlRootElement
+public class MetricAnomalyDetectorTestInput {
+
+ public MetricAnomalyDetectorTestInput() {
+ }
+
+ //Train data
+ private String trainDataName;
+ private String trainDataType;
+ private Map<String, String> trainDataConfigs;
+ private int trainDataSize;
+
+ //Test data
+ private String testDataName;
+ private String testDataType;
+ private Map<String, String> testDataConfigs;
+ private int testDataSize;
+
+ //Algorithm data
+ private List<String> methods;
+ private Map<String, String> methodConfigs;
+
+ public String getTrainDataName() {
+ return trainDataName;
+ }
+
+ public void setTrainDataName(String trainDataName) {
+ this.trainDataName = trainDataName;
+ }
+
+ public String getTrainDataType() {
+ return trainDataType;
+ }
+
+ public void setTrainDataType(String trainDataType) {
+ this.trainDataType = trainDataType;
+ }
+
+ public Map<String, String> getTrainDataConfigs() {
+ return trainDataConfigs;
+ }
+
+ public void setTrainDataConfigs(Map<String, String> trainDataConfigs) {
+ this.trainDataConfigs = trainDataConfigs;
+ }
+
+ public String getTestDataName() {
+ return testDataName;
+ }
+
+ public void setTestDataName(String testDataName) {
+ this.testDataName = testDataName;
+ }
+
+ public String getTestDataType() {
+ return testDataType;
+ }
+
+ public void setTestDataType(String testDataType) {
+ this.testDataType = testDataType;
+ }
+
+ public Map<String, String> getTestDataConfigs() {
+ return testDataConfigs;
+ }
+
+ public void setTestDataConfigs(Map<String, String> testDataConfigs) {
+ this.testDataConfigs = testDataConfigs;
+ }
+
+ public Map<String, String> getMethodConfigs() {
+ return methodConfigs;
+ }
+
+ public void setMethodConfigs(Map<String, String> methodConfigs) {
+ this.methodConfigs = methodConfigs;
+ }
+
+ public int getTrainDataSize() {
+ return trainDataSize;
+ }
+
+ public void setTrainDataSize(int trainDataSize) {
+ this.trainDataSize = trainDataSize;
+ }
+
+ public int getTestDataSize() {
+ return testDataSize;
+ }
+
+ public void setTestDataSize(int testDataSize) {
+ this.testDataSize = testDataSize;
+ }
+
+ public List<String> getMethods() {
+ return methods;
+ }
+
+ public void setMethods(List<String> methods) {
+ this.methods = methods;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/MetricAnomalyTester.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/MetricAnomalyTester.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/MetricAnomalyTester.java
new file mode 100644
index 0000000..d079e66
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/MetricAnomalyTester.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.adservice.prototype.testing.utilities;
+
+import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
+import org.apache.ambari.metrics.adservice.prototype.common.ResultSet;
+import org.apache.ambari.metrics.adservice.prototype.core.MetricsCollectorInterface;
+import org.apache.ambari.metrics.adservice.prototype.core.RFunctionInvoker;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Class which was originally used to send test series from AMS to Spark through Kafka.
+ */
+public class MetricAnomalyTester {
+
+// public static String appId = MetricsCollectorInterface.serviceName;
+// static final Log LOG = LogFactory.getLog(MetricAnomalyTester.class);
+// static Map<String, TimelineMetric> timelineMetricMap = new HashMap<>();
+//
+// public static TimelineMetrics runTestAnomalyRequest(MetricAnomalyDetectorTestInput input) throws UnknownHostException {
+//
+// long currentTime = System.currentTimeMillis();
+// TimelineMetrics timelineMetrics = new TimelineMetrics();
+// String hostname = InetAddress.getLocalHost().getHostName();
+//
+// //Train data
+// TimelineMetric metric1 = new TimelineMetric();
+// if (StringUtils.isNotEmpty(input.getTrainDataName())) {
+// metric1 = timelineMetricMap.get(input.getTrainDataName());
+// if (metric1 == null) {
+// metric1 = new TimelineMetric();
+// double[] trainSeries = MetricSeriesGeneratorFactory.generateSeries(input.getTrainDataType(), input.getTrainDataSize(), input.getTrainDataConfigs());
+// metric1.setMetricName(input.getTrainDataName());
+// metric1.setAppId(appId);
+// metric1.setHostName(hostname);
+// metric1.setStartTime(currentTime);
+// metric1.setInstanceId(null);
+// metric1.setMetricValues(getAsTimeSeries(currentTime, trainSeries));
+// timelineMetricMap.put(input.getTrainDataName(), metric1);
+// }
+// timelineMetrics.getMetrics().add(metric1);
+// } else {
+// LOG.error("No train data name specified");
+// }
+//
+// //Test data
+// TimelineMetric metric2 = new TimelineMetric();
+// if (StringUtils.isNotEmpty(input.getTestDataName())) {
+// metric2 = timelineMetricMap.get(input.getTestDataName());
+// if (metric2 == null) {
+// metric2 = new TimelineMetric();
+// double[] testSeries = MetricSeriesGeneratorFactory.generateSeries(input.getTestDataType(), input.getTestDataSize(), input.getTestDataConfigs());
+// metric2.setMetricName(input.getTestDataName());
+// metric2.setAppId(appId);
+// metric2.setHostName(hostname);
+// metric2.setStartTime(currentTime);
+// metric2.setInstanceId(null);
+// metric2.setMetricValues(getAsTimeSeries(currentTime, testSeries));
+// timelineMetricMap.put(input.getTestDataName(), metric2);
+// }
+// timelineMetrics.getMetrics().add(metric2);
+// } else {
+// LOG.warn("No test data name specified");
+// }
+//
+// //Invoke method
+// if (CollectionUtils.isNotEmpty(input.getMethods())) {
+// RFunctionInvoker.setScriptsDir("/etc/ambari-metrics-collector/conf/R-scripts");
+// for (String methodType : input.getMethods()) {
+// ResultSet result = RFunctionInvoker.executeMethod(methodType, getAsDataSeries(metric1), getAsDataSeries(metric2), input.getMethodConfigs());
+// TimelineMetric timelineMetric = getAsTimelineMetric(result, methodType, input, currentTime, hostname);
+// if (timelineMetric != null) {
+// timelineMetrics.getMetrics().add(timelineMetric);
+// }
+// }
+// } else {
+// LOG.warn("No anomaly method requested");
+// }
+//
+// return timelineMetrics;
+// }
+//
+//
+// private static TimelineMetric getAsTimelineMetric(ResultSet result, String methodType, MetricAnomalyDetectorTestInput input, long currentTime, String hostname) {
+//
+// if (result == null) {
+// return null;
+// }
+//
+// TimelineMetric timelineMetric = new TimelineMetric();
+// if (methodType.equals("tukeys") || methodType.equals("ema")) {
+// timelineMetric.setMetricName(input.getTrainDataName() + "_" + input.getTestDataName() + "_" + methodType + "_" + currentTime);
+// timelineMetric.setHostName(hostname);
+// timelineMetric.setAppId(appId);
+// timelineMetric.setInstanceId(null);
+// timelineMetric.setStartTime(currentTime);
+//
+// TreeMap<Long, Double> metricValues = new TreeMap<>();
+// if (result.resultset.size() > 0) {
+// double[] ts = result.resultset.get(0);
+// double[] metrics = result.resultset.get(1);
+// for (int i = 0; i < ts.length; i++) {
+// if (i == 0) {
+// timelineMetric.setStartTime((long) ts[i]);
+// }
+// metricValues.put((long) ts[i], metrics[i]);
+// }
+// }
+// timelineMetric.setMetricValues(metricValues);
+// return timelineMetric;
+// }
+// return null;
+// }
+//
+//
+// private static TreeMap<Long, Double> getAsTimeSeries(long currentTime, double[] values) {
+//
+// long startTime = currentTime - (values.length - 1) * 60 * 1000;
+// TreeMap<Long, Double> metricValues = new TreeMap<>();
+//
+// for (int i = 0; i < values.length; i++) {
+// metricValues.put(startTime, values[i]);
+// startTime += (60 * 1000);
+// }
+// return metricValues;
+// }
+//
+// private static DataSeries getAsDataSeries(TimelineMetric timelineMetric) {
+//
+// TreeMap<Long, Double> metricValues = timelineMetric.getMetricValues();
+// double[] timestamps = new double[metricValues.size()];
+// double[] values = new double[metricValues.size()];
+// int i = 0;
+//
+// for (Long timestamp : metricValues.keySet()) {
+// timestamps[i] = timestamp;
+// values[i++] = metricValues.get(timestamp);
+// }
+// return new DataSeries(timelineMetric.getMetricName() + "_" + timelineMetric.getAppId() + "_" + timelineMetric.getHostName(), timestamps, values);
+// }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/TestMetricSeriesGenerator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/TestMetricSeriesGenerator.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/TestMetricSeriesGenerator.java
new file mode 100644
index 0000000..3b2605b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/TestMetricSeriesGenerator.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.testing.utilities;
+
+/**
+ * Class which was originally used to send test series from AMS to Spark through Kafka.
+ */
+
+public class TestMetricSeriesGenerator {
+ //implements Runnable {
+
+// private Map<TestSeriesInputRequest, AbstractMetricSeries> configuredSeries = new HashMap<>();
+// private static final Log LOG = LogFactory.getLog(TestMetricSeriesGenerator.class);
+// private TimelineMetricStore metricStore;
+// private String hostname;
+//
+// public TestMetricSeriesGenerator(TimelineMetricStore metricStore) {
+// this.metricStore = metricStore;
+// try {
+// this.hostname = InetAddress.getLocalHost().getHostName();
+// } catch (UnknownHostException e) {
+// e.printStackTrace();
+// }
+// }
+//
+// public void addSeries(TestSeriesInputRequest inputRequest) {
+// if (!configuredSeries.containsKey(inputRequest)) {
+// AbstractMetricSeries metricSeries = MetricSeriesGeneratorFactory.generateSeries(inputRequest.getSeriesType(), inputRequest.getConfigs());
+// configuredSeries.put(inputRequest, metricSeries);
+// LOG.info("Added series " + inputRequest.getSeriesName());
+// }
+// }
+//
+// public void removeSeries(String seriesName) {
+// boolean isPresent = false;
+// TestSeriesInputRequest tbd = null;
+// for (TestSeriesInputRequest inputRequest : configuredSeries.keySet()) {
+// if (inputRequest.getSeriesName().equals(seriesName)) {
+// isPresent = true;
+// tbd = inputRequest;
+// }
+// }
+// if (isPresent) {
+// LOG.info("Removing series " + seriesName);
+// configuredSeries.remove(tbd);
+// } else {
+// LOG.info("Series not found : " + seriesName);
+// }
+// }
+//
+// @Override
+// public void run() {
+// long currentTime = System.currentTimeMillis();
+// TimelineMetrics timelineMetrics = new TimelineMetrics();
+//
+// for (TestSeriesInputRequest input : configuredSeries.keySet()) {
+// AbstractMetricSeries metricSeries = configuredSeries.get(input);
+// TimelineMetric timelineMetric = new TimelineMetric();
+// timelineMetric.setMetricName(input.getSeriesName());
+// timelineMetric.setAppId("anomaly-engine-test-metric");
+// timelineMetric.setInstanceId(null);
+// timelineMetric.setStartTime(currentTime);
+// timelineMetric.setHostName(hostname);
+// TreeMap<Long, Double> metricValues = new TreeMap();
+// metricValues.put(currentTime, metricSeries.nextValue());
+// timelineMetric.setMetricValues(metricValues);
+// timelineMetrics.addOrMergeTimelineMetric(timelineMetric);
+// LOG.info("Emitting metric with appId = " + timelineMetric.getAppId());
+// }
+// try {
+// LOG.info("Publishing test metrics for " + timelineMetrics.getMetrics().size() + " series.");
+// metricStore.putMetrics(timelineMetrics);
+// } catch (Exception e) {
+// LOG.error(e);
+// }
+// }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/TestSeriesInputRequest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/TestSeriesInputRequest.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/TestSeriesInputRequest.java
new file mode 100644
index 0000000..d7db9ca
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/TestSeriesInputRequest.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.testing.utilities;
+
+import org.apache.htrace.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper;
+
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.Collections;
+import java.util.Map;
+
+@XmlRootElement
+public class TestSeriesInputRequest {
+
+ private String seriesName;
+ private String seriesType;
+ private Map<String, String> configs;
+
+ public TestSeriesInputRequest() {
+ }
+
+ public TestSeriesInputRequest(String seriesName, String seriesType, Map<String, String> configs) {
+ this.seriesName = seriesName;
+ this.seriesType = seriesType;
+ this.configs = configs;
+ }
+
+ public String getSeriesName() {
+ return seriesName;
+ }
+
+ public void setSeriesName(String seriesName) {
+ this.seriesName = seriesName;
+ }
+
+ public String getSeriesType() {
+ return seriesType;
+ }
+
+ public void setSeriesType(String seriesType) {
+ this.seriesType = seriesType;
+ }
+
+ public Map<String, String> getConfigs() {
+ return configs;
+ }
+
+ public void setConfigs(Map<String, String> configs) {
+ this.configs = configs;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ TestSeriesInputRequest anotherInput = (TestSeriesInputRequest)o;
+ return anotherInput.getSeriesName().equals(this.getSeriesName());
+ }
+
+ @Override
+ public int hashCode() {
+ return seriesName.hashCode();
+ }
+
+ public static void main(String[] args) {
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ TestSeriesInputRequest testSeriesInputRequest = new TestSeriesInputRequest("test", "ema", Collections.singletonMap("key","value"));
+ try {
+ System.out.print(objectMapper.writeValueAsString(testSeriesInputRequest));
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/ema.R
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/ema.R b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/ema.R
new file mode 100644
index 0000000..0b66095
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/ema.R
@@ -0,0 +1,96 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# EMA <- w * EMA + (1 - w) * x
+# EMS <- sqrt( w * EMS^2 + (1 - w) * (x - EMA)^2 )
+# Alarm = abs(x - EMA) > n * EMS
+
+ema_global <- function(train_data, test_data, w, n) {
+
+# res <- get_data(url)
+# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
+# names(data) <- c("TS", res$metrics[[1]]$metricname)
+# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2]
+# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
+
+ anomalies <- data.frame()
+ ema <- 0
+ ems <- 0
+
+ #Train Step
+ for (x in train_data) {
+ ema <- w*ema + (1-w)*x
+ ems <- sqrt(w* ems^2 + (1 - w)*(x - ema)^2)
+ }
+
+ for ( i in 1:length(test_data[,1])) {
+ x <- test_data[i,2]
+ if (abs(x - ema) > n*ems) {
+ anomaly <- c(as.numeric(test_data[i,1]), x)
+ # print (anomaly)
+ anomalies <- rbind(anomalies, anomaly)
+ }
+ ema <- w*ema + (1-w)*x
+ ems <- sqrt(w* ems^2 + (1 - w)*(x - ema)^2)
+ }
+
+ if(length(anomalies) > 0) {
+ names(anomalies) <- c("TS", "Value")
+ }
+ return (anomalies)
+}
+
+ema_daily <- function(train_data, test_data, w, n) {
+
+# res <- get_data(url)
+# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
+# names(data) <- c("TS", res$metrics[[1]]$metricname)
+# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), ]
+# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
+
+ anomalies <- data.frame()
+ ema <- vector("numeric", 7)
+ ems <- vector("numeric", 7)
+
+ #Train Step
+ for ( i in 1:length(train_data[,1])) {
+ x <- train_data[i,2]
+ time <- as.POSIXlt(as.numeric(train_data[i,1])/1000, origin = "1970-01-01" ,tz = "GMT")
+ index <- time$wday
+ ema[index] <- w*ema[index] + (1-w)*x
+ ems[index] <- sqrt(w* ems[index]^2 + (1 - w)*(x - ema[index])^2)
+ }
+
+ for ( i in 1:length(test_data[,1])) {
+ x <- test_data[i,2]
+ time <- as.POSIXlt(as.numeric(test_data[i,1])/1000, origin = "1970-01-01" ,tz = "GMT")
+ index <- time$wday
+
+ if (abs(x - ema[index+1]) > n*ems[index+1]) {
+ anomaly <- c(as.numeric(test_data[i,1]), x)
+ # print (anomaly)
+ anomalies <- rbind(anomalies, anomaly)
+ }
+ ema[index+1] <- w*ema[index+1] + (1-w)*x
+ ems[index+1] <- sqrt(w* ems[index+1]^2 + (1 - w)*(x - ema[index+1])^2)
+ }
+
+ if(length(anomalies) > 0) {
+ names(anomalies) <- c("TS", "Value")
+ }
+ return(anomalies)
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/hsdev.r
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/hsdev.r b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/hsdev.r
new file mode 100644
index 0000000..bca3366
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/hsdev.r
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hsdev_daily <- function(train_data, test_data, n, num_historic_periods, interval, period) {
+
+ #res <- get_data(url)
+ #data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
+ #names(data) <- c("TS", res$metrics[[1]]$metricname)
+ anomalies <- data.frame()
+
+ granularity <- train_data[2,1] - train_data[1,1]
+ test_start <- test_data[1,1]
+ test_end <- test_data[length(test_data[1,]),1]
+ train_start <- test_start - num_historic_periods*period
+ # round to start of day
+ train_start <- train_start - (train_start %% interval)
+
+ time <- as.POSIXlt(as.numeric(test_data[1,1])/1000, origin = "1970-01-01" ,tz = "GMT")
+ test_data_day <- time$wday
+
+ h_data <- c()
+ for ( i in length(train_data[,1]):1) {
+ ts <- train_data[i,1]
+ if ( ts < train_start) {
+ break
+ }
+ time <- as.POSIXlt(as.numeric(ts)/1000, origin = "1970-01-01" ,tz = "GMT")
+ if (time$wday == test_data_day) {
+ x <- train_data[i,2]
+ h_data <- c(h_data, x)
+ }
+ }
+
+ if (length(h_data) < 2*length(test_data[,1])) {
+ cat ("\nNot enough training data")
+ return (anomalies)
+ }
+
+ past_median <- median(h_data)
+ past_sd <- sd(h_data)
+ curr_median <- median(test_data[,2])
+
+ if (abs(curr_median - past_median) > n * past_sd) {
+ anomaly <- c(test_start, test_end, curr_median, past_median, past_sd)
+ anomalies <- rbind(anomalies, anomaly)
+ }
+
+ if(length(anomalies) > 0) {
+ names(anomalies) <- c("TS Start", "TS End", "Current Median", "Past Median", "Past SD")
+ }
+
+ return (anomalies)
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/iforest.R
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/iforest.R b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/iforest.R
new file mode 100644
index 0000000..8956400
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/iforest.R
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ams_iforest <- function(url, train_start, train_end, test_start, test_end, threshold_score) {
+
+ res <- get_data(url)
+ num_metrics <- length(res$metrics)
+ anomalies <- data.frame()
+
+ metricname <- res$metrics[[1]]$metricname
+ data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
+ names(data) <- c("TS", res$metrics[[1]]$metricname)
+
+ for (i in 2:num_metrics) {
+ metricname <- res$metrics[[i]]$metricname
+ df <- data.frame(as.numeric(names(res$metrics[[i]]$metrics)), as.numeric(res$metrics[[i]]$metrics))
+ names(df) <- c("TS", res$metrics[[i]]$metricname)
+ data <- merge(data, df)
+ }
+
+ algo_data <- data[ which(df$TS >= train_start & df$TS <= train_end) , ][c(1:num_metrics+1)]
+ iForest <- IsolationTrees(algo_data)
+ test_data <- data[ which(df$TS >= test_start & df$TS <= test_end) , ]
+
+ if_res <- AnomalyScore(test_data[c(1:num_metrics+1)], iForest)
+ for (i in 1:length(if_res$outF)) {
+ index <- test_start+i-1
+ if (if_res$outF[i] > threshold_score) {
+ anomaly <- c(test_data[i,1], if_res$outF[i], if_res$pathLength[i])
+ anomalies <- rbind(anomalies, anomaly)
+ }
+ }
+
+ if(length(anomalies) > 0) {
+ names(anomalies) <- c("TS", "Anomaly Score", "Path length")
+ }
+ return (anomalies)
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/kstest.r
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/kstest.r b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/kstest.r
new file mode 100644
index 0000000..f22bc15
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/kstest.r
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ams_ks <- function(train_data, test_data, p_value) {
+
+# res <- get_data(url)
+# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
+# names(data) <- c("TS", res$metrics[[1]]$metricname)
+# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2]
+# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), 2]
+
+ anomalies <- data.frame()
+ res <- ks.test(train_data[,2], test_data[,2])
+
+ if (res[2] < p_value) {
+ anomaly <- c(test_data[1,1], test_data[length(test_data),1], res[1], res[2])
+ anomalies <- rbind(anomalies, anomaly)
+ }
+
+ if(length(anomalies) > 0) {
+ names(anomalies) <- c("TS Start", "TS End", "D", "p-value")
+ }
+ return (anomalies)
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/test.R
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/test.R b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/test.R
new file mode 100644
index 0000000..7650356
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/test.R
@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+tukeys_anomalies <- data.frame()
+ema_global_anomalies <- data.frame()
+ema_daily_anomalies <- data.frame()
+ks_anomalies <- data.frame()
+hsdev_anomalies <- data.frame()
+
+init <- function() {
+ tukeys_anomalies <- data.frame()
+ ema_global_anomalies <- data.frame()
+ ema_daily_anomalies <- data.frame()
+ ks_anomalies <- data.frame()
+ hsdev_anomalies <- data.frame()
+}
+
+test_methods <- function(data) {
+
+ init()
+ #res <- get_data(url)
+ #data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
+ #names(data) <- c("TS", res$metrics[[1]]$metricname)
+
+ limit <- data[length(data[,1]),1]
+ step <- data[2,1] - data[1,1]
+
+ train_start <- data[1,1]
+ train_end <- get_next_day_boundary(train_start, step, limit)
+ test_start <- train_end + step
+ test_end <- get_next_day_boundary(test_start, step, limit)
+ i <- 1
+ day <- 24*60*60*1000
+
+ while (test_start < limit) {
+
+ print (i)
+ i <- i + 1
+ train_data <- data[which(data$TS >= train_start & data$TS <= train_end),]
+ test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
+
+ #tukeys_anomalies <<- rbind(tukeys_anomalies, ams_tukeys(train_data, test_data, 3))
+ #ema_global_anomalies <<- rbind(ema_global_anomalies, ema_global(train_data, test_data, 0.9, 3))
+ #ema_daily_anomalies <<- rbind(ema_daily_anomalies, ema_daily(train_data, test_data, 0.9, 3))
+ #ks_anomalies <<- rbind(ks_anomalies, ams_ks(train_data, test_data, 0.05))
+ hsdev_train_data <- data[which(data$TS < test_start),]
+ hsdev_anomalies <<- rbind(hsdev_anomalies, hsdev_daily(hsdev_train_data, test_data, 3, 3, day, 7*day))
+
+ train_start <- test_start
+ train_end <- get_next_day_boundary(train_start, step, limit)
+ test_start <- train_end + step
+ test_end <- get_next_day_boundary(test_start, step, limit)
+ }
+ return (hsdev_anomalies)
+}
+
+get_next_day_boundary <- function(start, step, limit) {
+
+ if (start > limit) {
+ return (-1)
+ }
+
+ while (start <= limit) {
+ if (((start %% (24*60*60*1000)) - 28800000) == 0) {
+ return (start)
+ }
+ start <- start + step
+ }
+ return (start)
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/tukeys.r
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/tukeys.r b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/tukeys.r
new file mode 100644
index 0000000..0312226
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/R-scripts/tukeys.r
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+ams_tukeys <- function(train_data, test_data, n) {
+
+# res <- get_data(url)
+# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
+# names(data) <- c("TS", res$metrics[[1]]$metricname)
+# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2]
+# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
+
+ anomalies <- data.frame()
+ quantiles <- quantile(train_data[,2])
+ iqr <- quantiles[4] - quantiles[2]
+ niqr <- 0
+
+ for ( i in 1:length(test_data[,1])) {
+ x <- test_data[i,2]
+ lb <- quantiles[2] - n*iqr
+ ub <- quantiles[4] + n*iqr
+ if ( (x < lb) || (x > ub) ) {
+ if (iqr != 0) {
+ if (x < lb) {
+ niqr <- (quantiles[2] - x) / iqr
+ } else {
+ niqr <- (x - quantiles[4]) / iqr
+ }
+ }
+ anomaly <- c(test_data[i,1], x, niqr)
+ anomalies <- rbind(anomalies, anomaly)
+ }
+ }
+ if(length(anomalies) > 0) {
+ names(anomalies) <- c("TS", "Value", "niqr")
+ }
+ return (anomalies)
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/input-config.properties
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/input-config.properties b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/input-config.properties
new file mode 100644
index 0000000..ab106c4
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/input-config.properties
@@ -0,0 +1,42 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+appIds=HOST
+
+collectorHost=localhost
+collectorPort=6188
+collectorProtocol=http
+
+zkQuorum=localhost:2181
+
+ambariServerHost=localhost
+clusterName=c1
+
+emaW=0.8
+emaN=3
+tukeysN=3
+pointInTimeTestInterval=300000
+pointInTimeTrainInterval=900000
+
+ksTestInterval=600000
+ksTrainInterval=600000
+hsdevNhp=3
+hsdevInterval=1800000;
+
+skipMetricPatterns=sdisk*,cpu_sintr*,proc*,disk*,boottime
+hosts=avijayan-ad-1.openstacklocal
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/spark/prototype/MetricAnomalyDetector.scala
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/spark/prototype/MetricAnomalyDetector.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/spark/prototype/MetricAnomalyDetector.scala
new file mode 100644
index 0000000..6122f5e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/spark/prototype/MetricAnomalyDetector.scala
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.spark.prototype
+
+import java.io.{FileInputStream, IOException, InputStream}
+import java.util
+import java.util.Properties
+import java.util.logging.LogManager
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import org.apache.ambari.metrics.adservice.prototype.core.MetricsCollectorInterface
+import org.apache.spark.SparkConf
+import org.apache.spark.streaming._
+import org.apache.spark.streaming.kafka._
+import org.apache.ambari.metrics.adservice.prototype.methods.{AnomalyDetectionTechnique, MetricAnomaly}
+import org.apache.ambari.metrics.adservice.prototype.methods.ema.{EmaModelLoader, EmaTechnique}
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics
+import org.apache.log4j.Logger
+import org.apache.spark.storage.StorageLevel
+
+object MetricAnomalyDetector {
+
+ /*
+ Load current EMA model
+ Filter step - Check if anomaly
+ Collect / Write to AMS / Print.
+ */
+
+// var brokers = "avijayan-ams-1.openstacklocal:2181,avijayan-ams-2.openstacklocal:2181,avijayan-ams-3.openstacklocal:2181"
+// var groupId = "ambari-metrics-group"
+// var topicName = "ambari-metrics-topic"
+// var numThreads = 1
+// val anomalyDetectionModels: Array[AnomalyDetectionTechnique] = Array[AnomalyDetectionTechnique]()
+//
+// def readProperties(propertiesFile: String): Properties = try {
+// val properties = new Properties
+// var inputStream = ClassLoader.getSystemResourceAsStream(propertiesFile)
+// if (inputStream == null) inputStream = new FileInputStream(propertiesFile)
+// properties.load(inputStream)
+// properties
+// } catch {
+// case ioEx: IOException =>
+// null
+// }
+//
+// def main(args: Array[String]): Unit = {
+//
+// @transient
+// lazy val log = org.apache.log4j.LogManager.getLogger("MetricAnomalyDetectorLogger")
+//
+// if (args.length < 1) {
+// System.err.println("Usage: MetricSparkConsumer <input-config-file>")
+// System.exit(1)
+// }
+//
+// //Read properties
+// val properties = readProperties(propertiesFile = args(0))
+//
+// //Load EMA parameters - w, n
+// val emaW = properties.getProperty("emaW").toDouble
+// val emaN = properties.getProperty("emaN").toDouble
+//
+// //collector info
+// val collectorHost: String = properties.getProperty("collectorHost")
+// val collectorPort: String = properties.getProperty("collectorPort")
+// val collectorProtocol: String = properties.getProperty("collectorProtocol")
+// val anomalyMetricPublisher = new MetricsCollectorInterface(collectorHost, collectorProtocol, collectorPort)
+//
+// //Instantiate Kafka stream reader
+// val sparkConf = new SparkConf().setAppName("AmbariMetricsAnomalyDetector")
+// val streamingContext = new StreamingContext(sparkConf, Duration(10000))
+//
+// val topicsSet = topicName.toSet
+// val kafkaParams = Map[String, String]("metadata.broker.list" -> brokers)
+//// val stream = KafkaUtils.createDirectStream()
+//
+// val kafkaStream = KafkaUtils.createStream(streamingContext, zkQuorum, groupId, Map(topicName -> numThreads), StorageLevel.MEMORY_AND_DISK_SER_2)
+// kafkaStream.print()
+//
+// var timelineMetricsStream = kafkaStream.map( message => {
+// val mapper = new ObjectMapper
+// val metrics = mapper.readValue(message._2, classOf[TimelineMetrics])
+// metrics
+// })
+// timelineMetricsStream.print()
+//
+// var appMetricStream = timelineMetricsStream.map( timelineMetrics => {
+// (timelineMetrics.getMetrics.get(0).getAppId, timelineMetrics)
+// })
+// appMetricStream.print()
+//
+// var filteredAppMetricStream = appMetricStream.filter( appMetricTuple => {
+// appIds.contains(appMetricTuple._1)
+// } )
+// filteredAppMetricStream.print()
+//
+// filteredAppMetricStream.foreachRDD( rdd => {
+// rdd.foreach( appMetricTuple => {
+// val timelineMetrics = appMetricTuple._2
+// logger.info("Received Metric (1): " + timelineMetrics.getMetrics.get(0).getMetricName)
+// log.info("Received Metric (2): " + timelineMetrics.getMetrics.get(0).getMetricName)
+// for (timelineMetric <- timelineMetrics.getMetrics) {
+// var anomalies = emaModel.test(timelineMetric)
+// anomalyMetricPublisher.publish(anomalies)
+// }
+// })
+// })
+//
+// streamingContext.start()
+// streamingContext.awaitTermination()
+// }
+ }
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/spark/prototype/SparkPhoenixReader.scala
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/spark/prototype/SparkPhoenixReader.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/spark/prototype/SparkPhoenixReader.scala
new file mode 100644
index 0000000..6e1ae07
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/spark/prototype/SparkPhoenixReader.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.adservice.spark.prototype
+
+import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaTechnique
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.{SparkConf, SparkContext}
+
+object SparkPhoenixReader {
+
+ def main(args: Array[String]) {
+
+ if (args.length < 6) {
+ System.err.println("Usage: SparkPhoenixReader <metric_name> <appId> <hostname> <weight> <timessdev> <phoenixConnectionString> <model_dir>")
+ System.exit(1)
+ }
+
+ var metricName = args(0)
+ var appId = args(1)
+ var hostname = args(2)
+ var weight = args(3).toDouble
+ var timessdev = args(4).toInt
+ var phoenixConnectionString = args(5) //avijayan-ams-3.openstacklocal:61181:/ams-hbase-unsecure
+ var modelDir = args(6)
+
+ val conf = new SparkConf()
+ conf.set("spark.app.name", "AMSAnomalyModelBuilder")
+ //conf.set("spark.master", "spark://avijayan-ams-2.openstacklocal:7077")
+
+ var sc = new SparkContext(conf)
+ val sqlContext = new SQLContext(sc)
+
+ val currentTime = System.currentTimeMillis()
+ val oneDayBack = currentTime - 24*60*60*1000
+
+ val df = sqlContext.load("org.apache.phoenix.spark", Map("table" -> "METRIC_RECORD", "zkUrl" -> phoenixConnectionString))
+ df.registerTempTable("METRIC_RECORD")
+ val result = sqlContext.sql("SELECT METRIC_NAME, HOSTNAME, APP_ID, SERVER_TIME, METRIC_SUM, METRIC_COUNT FROM METRIC_RECORD " +
+ "WHERE METRIC_NAME = '" + metricName + "' AND HOSTNAME = '" + hostname + "' AND APP_ID = '" + appId + "' AND SERVER_TIME < " + currentTime + " AND SERVER_TIME > " + oneDayBack)
+
+ var metricValues = new java.util.TreeMap[java.lang.Long, java.lang.Double]
+ result.collect().foreach(
+ t => metricValues.put(t.getLong(3), t.getDouble(4) / t.getInt(5))
+ )
+
+ //val seriesName = result.head().getString(0)
+ //val hostname = result.head().getString(1)
+ //val appId = result.head().getString(2)
+
+ val timelineMetric = new TimelineMetric()
+ timelineMetric.setMetricName(metricName)
+ timelineMetric.setAppId(appId)
+ timelineMetric.setHostName(hostname)
+ timelineMetric.setMetricValues(metricValues)
+
+ var emaModel = new EmaTechnique(weight, timessdev)
+ emaModel.test(timelineMetric)
+ emaModel.save(sc, modelDir)
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestEmaTechnique.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestEmaTechnique.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestEmaTechnique.java
new file mode 100644
index 0000000..76a00a6
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestEmaTechnique.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype;
+
+import org.apache.ambari.metrics.adservice.prototype.core.RFunctionInvoker;
+import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
+import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaTechnique;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.List;
+import java.util.TreeMap;
+
+import static org.apache.ambari.metrics.adservice.prototype.TestRFunctionInvoker.getTS;
+
+public class TestEmaTechnique {
+
+ private static double[] ts;
+ private static String fullFilePath;
+
+ @BeforeClass
+ public static void init() throws URISyntaxException {
+
+ Assume.assumeTrue(System.getenv("R_HOME") != null);
+ ts = getTS(1000);
+ URL url = ClassLoader.getSystemResource("R-scripts");
+ fullFilePath = new File(url.toURI()).getAbsolutePath();
+ RFunctionInvoker.setScriptsDir(fullFilePath);
+ }
+
+ @Test
+ public void testEmaInitialization() {
+
+ EmaTechnique ema = new EmaTechnique(0.5, 3);
+ Assert.assertTrue(ema.getTrackedEmas().isEmpty());
+ Assert.assertTrue(ema.getStartingWeight() == 0.5);
+ Assert.assertTrue(ema.getStartTimesSdev() == 2);
+ }
+
+ @Test
+ public void testEma() {
+ EmaTechnique ema = new EmaTechnique(0.5, 3);
+
+ long now = System.currentTimeMillis();
+
+ TimelineMetric metric1 = new TimelineMetric();
+ metric1.setMetricName("M1");
+ metric1.setHostName("H1");
+ metric1.setStartTime(now - 1000);
+ metric1.setAppId("A1");
+ metric1.setInstanceId(null);
+ metric1.setType("Integer");
+
+ //Train
+ TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+ for (int i = 0; i < 50; i++) {
+ double metric = 20000 + Math.random();
+ metricValues.put(now - i * 100, metric);
+ }
+ metric1.setMetricValues(metricValues);
+ List<MetricAnomaly> anomalyList = ema.test(metric1);
+// Assert.assertTrue(anomalyList.isEmpty());
+
+ metricValues = new TreeMap<Long, Double>();
+ for (int i = 0; i < 50; i++) {
+ double metric = 20000 + Math.random();
+ metricValues.put(now - i * 100, metric);
+ }
+ metric1.setMetricValues(metricValues);
+ anomalyList = ema.test(metric1);
+ Assert.assertTrue(!anomalyList.isEmpty());
+ int l1 = anomalyList.size();
+
+ Assert.assertTrue(ema.updateModel(metric1, false, 20));
+ anomalyList = ema.test(metric1);
+ int l2 = anomalyList.size();
+ Assert.assertTrue(l2 < l1);
+
+ Assert.assertTrue(ema.updateModel(metric1, true, 50));
+ anomalyList = ema.test(metric1);
+ int l3 = anomalyList.size();
+ Assert.assertTrue(l3 > l2 && l3 > l1);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestRFunctionInvoker.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestRFunctionInvoker.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestRFunctionInvoker.java
new file mode 100644
index 0000000..98fa050
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestRFunctionInvoker.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype;
+
+import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
+import org.apache.ambari.metrics.adservice.prototype.common.ResultSet;
+import org.apache.ambari.metrics.adservice.prototype.core.RFunctionInvoker;
+import org.apache.ambari.metrics.adservice.seriesgenerator.UniformMetricSeries;
+import org.apache.commons.lang.ArrayUtils;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TestRFunctionInvoker {
+
+ private static String metricName = "TestMetric";
+ private static double[] ts;
+ private static String fullFilePath;
+
+ @BeforeClass
+ public static void init() throws URISyntaxException {
+
+ Assume.assumeTrue(System.getenv("R_HOME") != null);
+ ts = getTS(1000);
+ URL url = ClassLoader.getSystemResource("R-scripts");
+ fullFilePath = new File(url.toURI()).getAbsolutePath();
+ RFunctionInvoker.setScriptsDir(fullFilePath);
+ }
+
+ @Test
+ public void testTukeys() throws URISyntaxException {
+
+ double[] train_ts = ArrayUtils.subarray(ts, 0, 750);
+ double[] train_x = getRandomData(750);
+ DataSeries trainData = new DataSeries(metricName, train_ts, train_x);
+
+ double[] test_ts = ArrayUtils.subarray(ts, 750, 1000);
+ double[] test_x = getRandomData(250);
+ test_x[50] = 5.5; //Anomaly
+ DataSeries testData = new DataSeries(metricName, test_ts, test_x);
+ Map<String, String> configs = new HashMap();
+ configs.put("tukeys.n", "3");
+
+ ResultSet rs = RFunctionInvoker.tukeys(trainData, testData, configs);
+ Assert.assertEquals(rs.resultset.size(), 2);
+ Assert.assertEquals(rs.resultset.get(1)[0], 5.5, 0.1);
+
+ }
+
+ public static void main(String[] args) throws URISyntaxException {
+
+ String metricName = "TestMetric";
+ double[] ts = getTS(1000);
+ URL url = ClassLoader.getSystemResource("R-scripts");
+ String fullFilePath = new File(url.toURI()).getAbsolutePath();
+ RFunctionInvoker.setScriptsDir(fullFilePath);
+
+ double[] train_ts = ArrayUtils.subarray(ts, 0, 750);
+ double[] train_x = getRandomData(750);
+ DataSeries trainData = new DataSeries(metricName, train_ts, train_x);
+
+ double[] test_ts = ArrayUtils.subarray(ts, 750, 1000);
+ double[] test_x = getRandomData(250);
+ test_x[50] = 5.5; //Anomaly
+ DataSeries testData = new DataSeries(metricName, test_ts, test_x);
+ ResultSet rs;
+
+ Map<String, String> configs = new HashMap();
+
+ System.out.println("TUKEYS");
+ configs.put("tukeys.n", "3");
+ rs = RFunctionInvoker.tukeys(trainData, testData, configs);
+ rs.print();
+ System.out.println("--------------");
+
+// System.out.println("EMA Global");
+// configs.put("ema.n", "3");
+// configs.put("ema.w", "0.8");
+// rs = RFunctionInvoker.ema_global(trainData, testData, configs);
+// rs.print();
+// System.out.println("--------------");
+//
+// System.out.println("EMA Daily");
+// rs = RFunctionInvoker.ema_daily(trainData, testData, configs);
+// rs.print();
+// System.out.println("--------------");
+//
+// configs.put("ks.p_value", "0.00005");
+// System.out.println("KS Test");
+// rs = RFunctionInvoker.ksTest(trainData, testData, configs);
+// rs.print();
+// System.out.println("--------------");
+//
+ ts = getTS(5000);
+ train_ts = ArrayUtils.subarray(ts, 0, 4800);
+ train_x = getRandomData(4800);
+ trainData = new DataSeries(metricName, train_ts, train_x);
+ test_ts = ArrayUtils.subarray(ts, 4800, 5000);
+ test_x = getRandomData(200);
+ for (int i = 0; i < 200; i++) {
+ test_x[i] = test_x[i] * 5;
+ }
+ testData = new DataSeries(metricName, test_ts, test_x);
+ configs.put("hsdev.n", "3");
+ configs.put("hsdev.nhp", "3");
+ configs.put("hsdev.interval", "86400000");
+ configs.put("hsdev.period", "604800000");
+ System.out.println("HSdev");
+ rs = RFunctionInvoker.hsdev(trainData, testData, configs);
+ rs.print();
+ System.out.println("--------------");
+
+ }
+
+ static double[] getTS(int n) {
+ long currentTime = System.currentTimeMillis();
+ double[] ts = new double[n];
+ currentTime = currentTime - (currentTime % (5 * 60 * 1000));
+
+ for (int i = 0, j = n - 1; i < n; i++, j--) {
+ ts[j] = currentTime;
+ currentTime = currentTime - (5 * 60 * 1000);
+ }
+ return ts;
+ }
+
+ static double[] getRandomData(int n) {
+
+ UniformMetricSeries metricSeries = new UniformMetricSeries(10, 0.1,0.05, 0.6, 0.8, true);
+ return metricSeries.getSeries(n);
+
+// double[] metrics = new double[n];
+// Random random = new Random();
+// for (int i = 0; i < n; i++) {
+// metrics[i] = random.nextDouble();
+// }
+// return metrics;
+ }
+}
[3/6] ambari git commit: AMBARI-22077 : Create maven module and
package structure for the anomaly detection engine. (Commit 2) (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricSparkConsumer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricSparkConsumer.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricSparkConsumer.java
deleted file mode 100644
index 706c69f..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricSparkConsumer.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.core;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.ambari.metrics.alertservice.prototype.methods.MetricAnomaly;
-import org.apache.ambari.metrics.alertservice.prototype.methods.ema.EmaTechnique;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.broadcast.Broadcast;
-import org.apache.spark.streaming.Duration;
-import org.apache.spark.streaming.api.java.JavaDStream;
-import org.apache.spark.streaming.api.java.JavaPairDStream;
-import org.apache.spark.streaming.api.java.JavaPairReceiverInputDStream;
-import org.apache.spark.streaming.api.java.JavaStreamingContext;
-import org.apache.spark.streaming.kafka.KafkaUtils;
-import scala.Tuple2;
-
-import java.util.*;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class MetricSparkConsumer {
-
- private static final Log LOG = LogFactory.getLog(MetricSparkConsumer.class);
- private static String groupId = "ambari-metrics-group";
- private static String topicName = "ambari-metrics-topic";
- private static int numThreads = 1;
- private static long pitStartTime = System.currentTimeMillis();
- private static long ksStartTime = pitStartTime;
- private static long hdevStartTime = ksStartTime;
- private static Set<Pattern> includeMetricPatterns = new HashSet<>();
- private static Set<String> includedHosts = new HashSet<>();
- private static Set<TrendMetric> trendMetrics = new HashSet<>();
-
- public MetricSparkConsumer() {
- }
-
- public static Properties readProperties(String propertiesFile) {
- try {
- Properties properties = new Properties();
- InputStream inputStream = ClassLoader.getSystemResourceAsStream(propertiesFile);
- if (inputStream == null) {
- inputStream = new FileInputStream(propertiesFile);
- }
- properties.load(inputStream);
- return properties;
- } catch (IOException ioEx) {
- LOG.error("Error reading properties file for jmeter");
- return null;
- }
- }
-
- public static void main(String[] args) throws InterruptedException {
-
- if (args.length < 1) {
- System.err.println("Usage: MetricSparkConsumer <input-config-file>");
- System.exit(1);
- }
-
- Properties properties = readProperties(args[0]);
-
- List<String> appIds = Arrays.asList(properties.getProperty("appIds").split(","));
-
- String collectorHost = properties.getProperty("collectorHost");
- String collectorPort = properties.getProperty("collectorPort");
- String collectorProtocol = properties.getProperty("collectorProtocol");
-
- String zkQuorum = properties.getProperty("zkQuorum");
-
- double emaW = Double.parseDouble(properties.getProperty("emaW"));
- double emaN = Double.parseDouble(properties.getProperty("emaN"));
- int emaThreshold = Integer.parseInt(properties.getProperty("emaThreshold"));
- double tukeysN = Double.parseDouble(properties.getProperty("tukeysN"));
-
- long pitTestInterval = Long.parseLong(properties.getProperty("pointInTimeTestInterval"));
- long pitTrainInterval = Long.parseLong(properties.getProperty("pointInTimeTrainInterval"));
-
- long ksTestInterval = Long.parseLong(properties.getProperty("ksTestInterval"));
- long ksTrainInterval = Long.parseLong(properties.getProperty("ksTrainInterval"));
- int hsdevNhp = Integer.parseInt(properties.getProperty("hsdevNhp"));
- long hsdevInterval = Long.parseLong(properties.getProperty("hsdevInterval"));
-
- String ambariServerHost = properties.getProperty("ambariServerHost");
- String clusterName = properties.getProperty("clusterName");
-
- String includeMetricPatternStrings = properties.getProperty("includeMetricPatterns");
- if (includeMetricPatternStrings != null && !includeMetricPatternStrings.isEmpty()) {
- String[] patterns = includeMetricPatternStrings.split(",");
- for (String p : patterns) {
- LOG.info("Included Pattern : " + p);
- includeMetricPatterns.add(Pattern.compile(p));
- }
- }
-
- String includedHostList = properties.getProperty("hosts");
- if (includedHostList != null && !includedHostList.isEmpty()) {
- String[] hosts = includedHostList.split(",");
- includedHosts.addAll(Arrays.asList(hosts));
- }
-
- MetricsCollectorInterface metricsCollectorInterface = new MetricsCollectorInterface(collectorHost, collectorProtocol, collectorPort);
-
- SparkConf sparkConf = new SparkConf().setAppName("AmbariMetricsAnomalyDetector");
-
- JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, new Duration(10000));
-
- EmaTechnique emaTechnique = new EmaTechnique(emaW, emaN, emaThreshold);
- PointInTimeADSystem pointInTimeADSystem = new PointInTimeADSystem(metricsCollectorInterface,
- tukeysN,
- pitTestInterval,
- pitTrainInterval,
- ambariServerHost,
- clusterName);
-
- TrendADSystem trendADSystem = new TrendADSystem(metricsCollectorInterface,
- ksTestInterval,
- ksTrainInterval,
- hsdevNhp);
-
- Broadcast<EmaTechnique> emaTechniqueBroadcast = jssc.sparkContext().broadcast(emaTechnique);
- Broadcast<PointInTimeADSystem> pointInTimeADSystemBroadcast = jssc.sparkContext().broadcast(pointInTimeADSystem);
- Broadcast<TrendADSystem> trendADSystemBroadcast = jssc.sparkContext().broadcast(trendADSystem);
- Broadcast<MetricsCollectorInterface> metricsCollectorInterfaceBroadcast = jssc.sparkContext().broadcast(metricsCollectorInterface);
- Broadcast<Set<Pattern>> includePatternBroadcast = jssc.sparkContext().broadcast(includeMetricPatterns);
- Broadcast<Set<String>> includedHostBroadcast = jssc.sparkContext().broadcast(includedHosts);
-
- JavaPairReceiverInputDStream<String, String> messages =
- KafkaUtils.createStream(jssc, zkQuorum, groupId, Collections.singletonMap(topicName, numThreads));
-
- //Convert JSON string to TimelineMetrics.
- JavaDStream<TimelineMetrics> timelineMetricsStream = messages.map(new Function<Tuple2<String, String>, TimelineMetrics>() {
- @Override
- public TimelineMetrics call(Tuple2<String, String> message) throws Exception {
- ObjectMapper mapper = new ObjectMapper();
- TimelineMetrics metrics = mapper.readValue(message._2, TimelineMetrics.class);
- return metrics;
- }
- });
-
- timelineMetricsStream.print();
-
- //Group TimelineMetric by AppId.
- JavaPairDStream<String, TimelineMetrics> appMetricStream = timelineMetricsStream.mapToPair(
- timelineMetrics -> timelineMetrics.getMetrics().isEmpty() ? new Tuple2<>("TEST", new TimelineMetrics()) : new Tuple2<String, TimelineMetrics>(timelineMetrics.getMetrics().get(0).getAppId(), timelineMetrics)
- );
-
- appMetricStream.print();
-
- //Filter AppIds that are not needed.
- JavaPairDStream<String, TimelineMetrics> filteredAppMetricStream = appMetricStream.filter(new Function<Tuple2<String, TimelineMetrics>, Boolean>() {
- @Override
- public Boolean call(Tuple2<String, TimelineMetrics> appMetricTuple) throws Exception {
- return appIds.contains(appMetricTuple._1);
- }
- });
-
- filteredAppMetricStream.print();
-
- filteredAppMetricStream.foreachRDD(rdd -> {
- rdd.foreach(
- tuple2 -> {
- long currentTime = System.currentTimeMillis();
- EmaTechnique ema = emaTechniqueBroadcast.getValue();
- if (currentTime > pitStartTime + pitTestInterval) {
- LOG.info("Running Tukeys....");
- pointInTimeADSystemBroadcast.getValue().runTukeysAndRefineEma(ema, currentTime);
- pitStartTime = pitStartTime + pitTestInterval;
- }
-
- if (currentTime > ksStartTime + ksTestInterval) {
- LOG.info("Running KS Test....");
- trendADSystemBroadcast.getValue().runKSTest(currentTime, trendMetrics);
- ksStartTime = ksStartTime + ksTestInterval;
- }
-
- if (currentTime > hdevStartTime + hsdevInterval) {
- LOG.info("Running HSdev Test....");
- trendADSystemBroadcast.getValue().runHsdevMethod();
- hdevStartTime = hdevStartTime + hsdevInterval;
- }
-
- TimelineMetrics metrics = tuple2._2();
- for (TimelineMetric timelineMetric : metrics.getMetrics()) {
-
- boolean includeHost = includedHostBroadcast.getValue().contains(timelineMetric.getHostName());
- boolean includeMetric = false;
- if (includeHost) {
- if (includePatternBroadcast.getValue().isEmpty()) {
- includeMetric = true;
- }
- for (Pattern p : includePatternBroadcast.getValue()) {
- Matcher m = p.matcher(timelineMetric.getMetricName());
- if (m.find()) {
- includeMetric = true;
- }
- }
- }
-
- if (includeMetric) {
- trendMetrics.add(new TrendMetric(timelineMetric.getMetricName(), timelineMetric.getAppId(),
- timelineMetric.getHostName()));
- List<MetricAnomaly> anomalies = ema.test(timelineMetric);
- metricsCollectorInterfaceBroadcast.getValue().publish(anomalies);
- }
- }
- });
- });
-
- jssc.start();
- jssc.awaitTermination();
- }
-}
-
-
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricsCollectorInterface.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricsCollectorInterface.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricsCollectorInterface.java
deleted file mode 100644
index 246565d..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricsCollectorInterface.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.core;
-
-import org.apache.ambari.metrics.alertservice.prototype.methods.MetricAnomaly;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.codehaus.jackson.map.AnnotationIntrospector;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.ObjectReader;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.Serializable;
-import java.net.HttpURLConnection;
-import java.net.InetAddress;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.TreeMap;
-
-public class MetricsCollectorInterface implements Serializable {
-
- private static String hostName = null;
- private String instanceId = null;
- public final static String serviceName = "anomaly-engine";
- private String collectorHost;
- private String protocol;
- private String port;
- private static final String WS_V1_TIMELINE_METRICS = "/ws/v1/timeline/metrics";
- private static final Log LOG = LogFactory.getLog(MetricsCollectorInterface.class);
- private static ObjectMapper mapper;
- private final static ObjectReader timelineObjectReader;
-
- static {
- mapper = new ObjectMapper();
- AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
- mapper.setAnnotationIntrospector(introspector);
- mapper.getSerializationConfig()
- .withSerializationInclusion(JsonSerialize.Inclusion.NON_NULL);
- timelineObjectReader = mapper.reader(TimelineMetrics.class);
- }
-
- public MetricsCollectorInterface(String collectorHost, String protocol, String port) {
- this.collectorHost = collectorHost;
- this.protocol = protocol;
- this.port = port;
- this.hostName = getDefaultLocalHostName();
- }
-
- public static String getDefaultLocalHostName() {
-
- if (hostName != null) {
- return hostName;
- }
-
- try {
- return InetAddress.getLocalHost().getCanonicalHostName();
- } catch (UnknownHostException e) {
- LOG.info("Error getting host address");
- }
- return null;
- }
-
- public void publish(List<MetricAnomaly> metricAnomalies) {
- if (CollectionUtils.isNotEmpty(metricAnomalies)) {
- LOG.info("Sending metric anomalies of size : " + metricAnomalies.size());
- List<TimelineMetric> metricList = getTimelineMetricList(metricAnomalies);
- if (!metricList.isEmpty()) {
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- timelineMetrics.setMetrics(metricList);
- emitMetrics(timelineMetrics);
- }
- } else {
- LOG.debug("No anomalies to send.");
- }
- }
-
- private List<TimelineMetric> getTimelineMetricList(List<MetricAnomaly> metricAnomalies) {
- List<TimelineMetric> metrics = new ArrayList<>();
-
- if (metricAnomalies.isEmpty()) {
- return metrics;
- }
-
- for (MetricAnomaly anomaly : metricAnomalies) {
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName(anomaly.getMetricKey());
- timelineMetric.setAppId(serviceName + "-" + anomaly.getMethodType());
- timelineMetric.setInstanceId(null);
- timelineMetric.setHostName(getDefaultLocalHostName());
- timelineMetric.setStartTime(anomaly.getTimestamp());
- HashMap<String, String> metadata = new HashMap<>();
- metadata.put("method", anomaly.getMethodType());
- metadata.put("anomaly-score", String.valueOf(anomaly.getAnomalyScore()));
- timelineMetric.setMetadata(metadata);
- TreeMap<Long,Double> metricValues = new TreeMap<>();
- metricValues.put(anomaly.getTimestamp(), anomaly.getMetricValue());
- timelineMetric.setMetricValues(metricValues);
-
- metrics.add(timelineMetric);
- }
- return metrics;
- }
-
- public boolean emitMetrics(TimelineMetrics metrics) {
- String connectUrl = constructTimelineMetricUri();
- String jsonData = null;
- LOG.debug("EmitMetrics connectUrl = " + connectUrl);
- try {
- jsonData = mapper.writeValueAsString(metrics);
- LOG.info(jsonData);
- } catch (IOException e) {
- LOG.error("Unable to parse metrics", e);
- }
- if (jsonData != null) {
- return emitMetricsJson(connectUrl, jsonData);
- }
- return false;
- }
-
- private HttpURLConnection getConnection(String spec) throws IOException {
- return (HttpURLConnection) new URL(spec).openConnection();
- }
-
- private boolean emitMetricsJson(String connectUrl, String jsonData) {
- int timeout = 10000;
- HttpURLConnection connection = null;
- try {
- if (connectUrl == null) {
- throw new IOException("Unknown URL. Unable to connect to metrics collector.");
- }
- connection = getConnection(connectUrl);
-
- connection.setRequestMethod("POST");
- connection.setRequestProperty("Content-Type", "application/json");
- connection.setRequestProperty("Connection", "Keep-Alive");
- connection.setConnectTimeout(timeout);
- connection.setReadTimeout(timeout);
- connection.setDoOutput(true);
-
- if (jsonData != null) {
- try (OutputStream os = connection.getOutputStream()) {
- os.write(jsonData.getBytes("UTF-8"));
- }
- }
-
- int statusCode = connection.getResponseCode();
-
- if (statusCode != 200) {
- LOG.info("Unable to POST metrics to collector, " + connectUrl + ", " +
- "statusCode = " + statusCode);
- } else {
- LOG.info("Metrics posted to Collector " + connectUrl);
- }
- return true;
- } catch (IOException ioe) {
- LOG.error(ioe.getMessage());
- }
- return false;
- }
-
- private String constructTimelineMetricUri() {
- StringBuilder sb = new StringBuilder(protocol);
- sb.append("://");
- sb.append(collectorHost);
- sb.append(":");
- sb.append(port);
- sb.append(WS_V1_TIMELINE_METRICS);
- return sb.toString();
- }
-
- public TimelineMetrics fetchMetrics(String metricName,
- String appId,
- String hostname,
- long startime,
- long endtime) {
-
- String url = constructTimelineMetricUri() + "?metricNames=" + metricName + "&appId=" + appId +
- "&hostname=" + hostname + "&startTime=" + startime + "&endTime=" + endtime;
- LOG.debug("Fetch metrics URL : " + url);
-
- URL obj = null;
- BufferedReader in = null;
- TimelineMetrics timelineMetrics = new TimelineMetrics();
-
- try {
- obj = new URL(url);
- HttpURLConnection con = (HttpURLConnection) obj.openConnection();
- con.setRequestMethod("GET");
- int responseCode = con.getResponseCode();
- LOG.debug("Sending 'GET' request to URL : " + url);
- LOG.debug("Response Code : " + responseCode);
-
- in = new BufferedReader(
- new InputStreamReader(con.getInputStream()));
- timelineMetrics = timelineObjectReader.readValue(in);
- } catch (Exception e) {
- LOG.error(e);
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- LOG.warn(e);
- }
- }
- }
-
- LOG.info("Fetched " + timelineMetrics.getMetrics().size() + " metrics.");
- return timelineMetrics;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/PointInTimeADSystem.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/PointInTimeADSystem.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/PointInTimeADSystem.java
deleted file mode 100644
index c579515..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/PointInTimeADSystem.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.core;
-
-import org.apache.ambari.metrics.alertservice.prototype.common.DataSeries;
-import org.apache.ambari.metrics.alertservice.prototype.common.ResultSet;
-import org.apache.ambari.metrics.alertservice.prototype.methods.ema.EmaModel;
-import org.apache.ambari.metrics.alertservice.prototype.methods.ema.EmaTechnique;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-public class PointInTimeADSystem implements Serializable {
-
- //private EmaTechnique emaTechnique;
- private MetricsCollectorInterface metricsCollectorInterface;
- private Map<String, Double> tukeysNMap;
- private double defaultTukeysN = 3;
-
- private long testIntervalMillis = 5*60*1000; //10mins
- private long trainIntervalMillis = 15*60*1000; //1hour
-
- private static final Log LOG = LogFactory.getLog(PointInTimeADSystem.class);
-
- private AmbariServerInterface ambariServerInterface;
- private int sensitivity = 50;
- private int minSensitivity = 0;
- private int maxSensitivity = 100;
-
- public PointInTimeADSystem(MetricsCollectorInterface metricsCollectorInterface, double defaultTukeysN,
- long testIntervalMillis, long trainIntervalMillis, String ambariServerHost, String clusterName) {
- this.metricsCollectorInterface = metricsCollectorInterface;
- this.defaultTukeysN = defaultTukeysN;
- this.tukeysNMap = new HashMap<>();
- this.testIntervalMillis = testIntervalMillis;
- this.trainIntervalMillis = trainIntervalMillis;
- this.ambariServerInterface = new AmbariServerInterface(ambariServerHost, clusterName);
- LOG.info("Starting PointInTimeADSystem...");
- }
-
- public void runTukeysAndRefineEma(EmaTechnique emaTechnique, long startTime) {
- LOG.info("Running Tukeys for test data interval [" + new Date(startTime - testIntervalMillis) + " : " + new Date(startTime) + "], with train data period [" + new Date(startTime - testIntervalMillis - trainIntervalMillis) + " : " + new Date(startTime - testIntervalMillis) + "]");
-
- int requiredSensivity = ambariServerInterface.getPointInTimeSensitivity();
- if (requiredSensivity == -1 || requiredSensivity == sensitivity) {
- LOG.info("No change in sensitivity needed.");
- } else {
- LOG.info("Current tukey's N value = " + defaultTukeysN);
- if (requiredSensivity > sensitivity) {
- int targetSensitivity = Math.min(maxSensitivity, requiredSensivity);
- while (sensitivity < targetSensitivity) {
- defaultTukeysN = defaultTukeysN + defaultTukeysN * 0.05;
- sensitivity++;
- }
- } else {
- int targetSensitivity = Math.max(minSensitivity, requiredSensivity);
- while (sensitivity > targetSensitivity) {
- defaultTukeysN = defaultTukeysN - defaultTukeysN * 0.05;
- sensitivity--;
- }
- }
- LOG.info("New tukey's N value = " + defaultTukeysN);
- }
-
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- for (String metricKey : emaTechnique.getTrackedEmas().keySet()) {
- LOG.info("EMA key = " + metricKey);
- EmaModel emaModel = emaTechnique.getTrackedEmas().get(metricKey);
- String metricName = emaModel.getMetricName();
- String appId = emaModel.getAppId();
- String hostname = emaModel.getHostname();
-
- TimelineMetrics tukeysData = metricsCollectorInterface.fetchMetrics(metricName, appId, hostname, startTime - (testIntervalMillis + trainIntervalMillis),
- startTime);
-
- if (tukeysData.getMetrics().isEmpty()) {
- LOG.info("No metrics fetched for Tukeys, metricKey = " + metricKey);
- continue;
- }
-
- List<Double> trainTsList = new ArrayList<>();
- List<Double> trainDataList = new ArrayList<>();
- List<Double> testTsList = new ArrayList<>();
- List<Double> testDataList = new ArrayList<>();
-
- for (TimelineMetric metric : tukeysData.getMetrics()) {
- for (Long timestamp : metric.getMetricValues().keySet()) {
- if (timestamp <= (startTime - testIntervalMillis)) {
- trainDataList.add(metric.getMetricValues().get(timestamp));
- trainTsList.add((double)timestamp);
- } else {
- testDataList.add(metric.getMetricValues().get(timestamp));
- testTsList.add((double)timestamp);
- }
- }
- }
-
- if (trainDataList.isEmpty() || testDataList.isEmpty() || trainDataList.size() < testDataList.size()) {
- LOG.info("Not enough train/test data to perform analysis.");
- continue;
- }
-
- String tukeysTrainSeries = "tukeysTrainSeries";
- double[] trainTs = new double[trainTsList.size()];
- double[] trainData = new double[trainTsList.size()];
- for (int i = 0; i < trainTs.length; i++) {
- trainTs[i] = trainTsList.get(i);
- trainData[i] = trainDataList.get(i);
- }
-
- String tukeysTestSeries = "tukeysTestSeries";
- double[] testTs = new double[testTsList.size()];
- double[] testData = new double[testTsList.size()];
- for (int i = 0; i < testTs.length; i++) {
- testTs[i] = testTsList.get(i);
- testData[i] = testDataList.get(i);
- }
-
- LOG.info("Train Size = " + trainTs.length + ", Test Size = " + testTs.length);
-
- DataSeries tukeysTrainData = new DataSeries(tukeysTrainSeries, trainTs, trainData);
- DataSeries tukeysTestData = new DataSeries(tukeysTestSeries, testTs, testData);
-
- if (!tukeysNMap.containsKey(metricKey)) {
- tukeysNMap.put(metricKey, defaultTukeysN);
- }
-
- Map<String, String> configs = new HashMap<>();
- configs.put("tukeys.n", String.valueOf(tukeysNMap.get(metricKey)));
-
- ResultSet rs = RFunctionInvoker.tukeys(tukeysTrainData, tukeysTestData, configs);
-
- List<TimelineMetric> tukeysMetrics = getAsTimelineMetric(rs, metricName, appId, hostname);
- LOG.info("Tukeys anomalies size : " + tukeysMetrics.size());
- TreeMap<Long, Double> tukeysMetricValues = new TreeMap<>();
-
- for (TimelineMetric tukeysMetric : tukeysMetrics) {
- tukeysMetricValues.putAll(tukeysMetric.getMetricValues());
- timelineMetrics.addOrMergeTimelineMetric(tukeysMetric);
- }
-
- TimelineMetrics emaData = metricsCollectorInterface.fetchMetrics(metricKey, MetricsCollectorInterface.serviceName+"-ema", MetricsCollectorInterface.getDefaultLocalHostName(), startTime - testIntervalMillis, startTime);
- TreeMap<Long, Double> emaMetricValues = new TreeMap();
- if (!emaData.getMetrics().isEmpty()) {
- emaMetricValues = emaData.getMetrics().get(0).getMetricValues();
- }
-
- LOG.info("Ema anomalies size : " + emaMetricValues.size());
- int tp = 0;
- int tn = 0;
- int fp = 0;
- int fn = 0;
-
- for (double ts : testTs) {
- long timestamp = (long) ts;
- if (tukeysMetricValues.containsKey(timestamp)) {
- if (emaMetricValues.containsKey(timestamp)) {
- tp++;
- } else {
- fn++;
- }
- } else {
- if (emaMetricValues.containsKey(timestamp)) {
- fp++;
- } else {
- tn++;
- }
- }
- }
-
- double recall = (double) tp / (double) (tp + fn);
- double precision = (double) tp / (double) (tp + fp);
- LOG.info("----------------------------");
- LOG.info("Precision Recall values for " + metricKey);
- LOG.info("tp=" + tp + ", fp=" + fp + ", tn=" + tn + ", fn=" + fn);
- LOG.info("----------------------------");
-
- if (recall < 0.5) {
- LOG.info("Increasing EMA sensitivity by 10%");
- emaModel.updateModel(true, 5);
- } else if (precision < 0.5) {
- LOG.info("Decreasing EMA sensitivity by 10%");
- emaModel.updateModel(false, 5);
- }
-
- }
-
- if (emaTechnique.getTrackedEmas().isEmpty()){
- LOG.info("No EMA Technique keys tracked!!!!");
- }
-
- if (!timelineMetrics.getMetrics().isEmpty()) {
- metricsCollectorInterface.emitMetrics(timelineMetrics);
- }
- }
-
- private static List<TimelineMetric> getAsTimelineMetric(ResultSet result, String metricName, String appId, String hostname) {
-
- List<TimelineMetric> timelineMetrics = new ArrayList<>();
-
- if (result == null) {
- LOG.info("ResultSet from R call is null!!");
- return null;
- }
-
- if (result.resultset.size() > 0) {
- double[] ts = result.resultset.get(0);
- double[] metrics = result.resultset.get(1);
- double[] anomalyScore = result.resultset.get(2);
- for (int i = 0; i < ts.length; i++) {
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName(metricName + ":" + appId + ":" + hostname);
- timelineMetric.setHostName(MetricsCollectorInterface.getDefaultLocalHostName());
- timelineMetric.setAppId(MetricsCollectorInterface.serviceName + "-tukeys");
- timelineMetric.setInstanceId(null);
- timelineMetric.setStartTime((long) ts[i]);
- TreeMap<Long, Double> metricValues = new TreeMap<>();
- metricValues.put((long) ts[i], metrics[i]);
-
- HashMap<String, String> metadata = new HashMap<>();
- metadata.put("method", "tukeys");
- if (String.valueOf(anomalyScore[i]).equals("infinity")) {
- LOG.info("Got anomalyScore = infinity for " + metricName + ":" + appId + ":" + hostname);
- } else {
- metadata.put("anomaly-score", String.valueOf(anomalyScore[i]));
- }
- timelineMetric.setMetadata(metadata);
-
- timelineMetric.setMetricValues(metricValues);
- timelineMetrics.add(timelineMetric);
- }
- }
-
- return timelineMetrics;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/RFunctionInvoker.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/RFunctionInvoker.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/RFunctionInvoker.java
deleted file mode 100644
index 4538f0b..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/RFunctionInvoker.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.core;
-
-
-import org.apache.ambari.metrics.alertservice.prototype.common.ResultSet;
-import org.apache.ambari.metrics.alertservice.prototype.common.DataSeries;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.rosuda.JRI.REXP;
-import org.rosuda.JRI.RVector;
-import org.rosuda.JRI.Rengine;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class RFunctionInvoker {
-
- static final Log LOG = LogFactory.getLog(RFunctionInvoker.class);
- public static Rengine r = new Rengine(new String[]{"--no-save"}, false, null);
- private static String rScriptDir = "/usr/lib/ambari-metrics-collector/R-scripts";
-
- private static void loadDataSets(Rengine r, DataSeries trainData, DataSeries testData) {
- r.assign("train_ts", trainData.ts);
- r.assign("train_x", trainData.values);
- r.eval("train_data <- data.frame(train_ts,train_x)");
- r.eval("names(train_data) <- c(\"TS\", " + trainData.seriesName + ")");
-
- r.assign("test_ts", testData.ts);
- r.assign("test_x", testData.values);
- r.eval("test_data <- data.frame(test_ts,test_x)");
- r.eval("names(test_data) <- c(\"TS\", " + testData.seriesName + ")");
- }
-
- public static void setScriptsDir(String dir) {
- rScriptDir = dir;
- }
-
- public static ResultSet executeMethod(String methodType, DataSeries trainData, DataSeries testData, Map<String, String> configs) {
-
- ResultSet result;
- switch (methodType) {
- case "tukeys":
- result = tukeys(trainData, testData, configs);
- break;
- case "ema":
- result = ema_global(trainData, testData, configs);
- break;
- case "ks":
- result = ksTest(trainData, testData, configs);
- break;
- case "hsdev":
- result = hsdev(trainData, testData, configs);
- break;
- default:
- result = tukeys(trainData, testData, configs);
- break;
- }
- return result;
- }
-
- public static ResultSet tukeys(DataSeries trainData, DataSeries testData, Map<String, String> configs) {
- try {
-
- REXP exp1 = r.eval("source('" + rScriptDir + "/tukeys.r" + "')");
-
- double n = Double.parseDouble(configs.get("tukeys.n"));
- r.eval("n <- " + n);
-
- loadDataSets(r, trainData, testData);
-
- r.eval("an <- ams_tukeys(train_data, test_data, n)");
- REXP exp = r.eval("an");
- RVector cont = (RVector) exp.getContent();
- List<double[]> result = new ArrayList();
- for (int i = 0; i < cont.size(); i++) {
- result.add(cont.at(i).asDoubleArray());
- }
- return new ResultSet(result);
- } catch (Exception e) {
- LOG.error(e);
- } finally {
- r.end();
- }
- return null;
- }
-
- public static ResultSet ema_global(DataSeries trainData, DataSeries testData, Map<String, String> configs) {
- try {
- r.eval("source('" + rScriptDir + "/ema.r" + "')");
-
- int n = Integer.parseInt(configs.get("ema.n"));
- r.eval("n <- " + n);
-
- double w = Double.parseDouble(configs.get("ema.w"));
- r.eval("w <- " + w);
-
- loadDataSets(r, trainData, testData);
-
- r.eval("an <- ema_global(train_data, test_data, w, n)");
- REXP exp = r.eval("an");
- RVector cont = (RVector) exp.getContent();
- List<double[]> result = new ArrayList();
- for (int i = 0; i < cont.size(); i++) {
- result.add(cont.at(i).asDoubleArray());
- }
- return new ResultSet(result);
-
- } catch (Exception e) {
- LOG.error(e);
- } finally {
- r.end();
- }
- return null;
- }
-
- public static ResultSet ema_daily(DataSeries trainData, DataSeries testData, Map<String, String> configs) {
- try {
- r.eval("source('" + rScriptDir + "/ema.r" + "')");
-
- int n = Integer.parseInt(configs.get("ema.n"));
- r.eval("n <- " + n);
-
- double w = Double.parseDouble(configs.get("ema.w"));
- r.eval("w <- " + w);
-
- loadDataSets(r, trainData, testData);
-
- r.eval("an <- ema_daily(train_data, test_data, w, n)");
- REXP exp = r.eval("an");
- RVector cont = (RVector) exp.getContent();
- List<double[]> result = new ArrayList();
- for (int i = 0; i < cont.size(); i++) {
- result.add(cont.at(i).asDoubleArray());
- }
- return new ResultSet(result);
-
- } catch (Exception e) {
- LOG.error(e);
- } finally {
- r.end();
- }
- return null;
- }
-
- public static ResultSet ksTest(DataSeries trainData, DataSeries testData, Map<String, String> configs) {
- try {
- r.eval("source('" + rScriptDir + "/kstest.r" + "')");
-
- double p_value = Double.parseDouble(configs.get("ks.p_value"));
- r.eval("p_value <- " + p_value);
-
- loadDataSets(r, trainData, testData);
-
- r.eval("an <- ams_ks(train_data, test_data, p_value)");
- REXP exp = r.eval("an");
- RVector cont = (RVector) exp.getContent();
- List<double[]> result = new ArrayList();
- for (int i = 0; i < cont.size(); i++) {
- result.add(cont.at(i).asDoubleArray());
- }
- return new ResultSet(result);
-
- } catch (Exception e) {
- LOG.error(e);
- } finally {
- r.end();
- }
- return null;
- }
-
- public static ResultSet hsdev(DataSeries trainData, DataSeries testData, Map<String, String> configs) {
- try {
- r.eval("source('" + rScriptDir + "/hsdev.r" + "')");
-
- int n = Integer.parseInt(configs.get("hsdev.n"));
- r.eval("n <- " + n);
-
- int nhp = Integer.parseInt(configs.get("hsdev.nhp"));
- r.eval("nhp <- " + nhp);
-
- long interval = Long.parseLong(configs.get("hsdev.interval"));
- r.eval("interval <- " + interval);
-
- long period = Long.parseLong(configs.get("hsdev.period"));
- r.eval("period <- " + period);
-
- loadDataSets(r, trainData, testData);
-
- r.eval("an2 <- hsdev_daily(train_data, test_data, n, nhp, interval, period)");
- REXP exp = r.eval("an2");
- RVector cont = (RVector) exp.getContent();
-
- List<double[]> result = new ArrayList();
- for (int i = 0; i < cont.size(); i++) {
- result.add(cont.at(i).asDoubleArray());
- }
- return new ResultSet(result);
- } catch (Exception e) {
- LOG.error(e);
- } finally {
- r.end();
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/TrendADSystem.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/TrendADSystem.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/TrendADSystem.java
deleted file mode 100644
index 2a205d1..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/TrendADSystem.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.core;
-
-import org.apache.ambari.metrics.alertservice.prototype.common.DataSeries;
-import org.apache.ambari.metrics.alertservice.prototype.methods.MetricAnomaly;
-import org.apache.ambari.metrics.alertservice.prototype.methods.hsdev.HsdevTechnique;
-import org.apache.ambari.metrics.alertservice.prototype.methods.kstest.KSTechnique;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-public class TrendADSystem implements Serializable {
-
- private MetricsCollectorInterface metricsCollectorInterface;
- private List<TrendMetric> trendMetrics;
-
- private long ksTestIntervalMillis = 10 * 60 * 1000;
- private long ksTrainIntervalMillis = 10 * 60 * 1000;
- private KSTechnique ksTechnique;
-
- private HsdevTechnique hsdevTechnique;
- private int hsdevNumHistoricalPeriods = 3;
-
- private Map<KsSingleRunKey, MetricAnomaly> trackedKsAnomalies = new HashMap<>();
- private static final Log LOG = LogFactory.getLog(TrendADSystem.class);
- private String inputFile = "";
-
- public TrendADSystem(MetricsCollectorInterface metricsCollectorInterface,
- long ksTestIntervalMillis,
- long ksTrainIntervalMillis,
- int hsdevNumHistoricalPeriods) {
-
- this.metricsCollectorInterface = metricsCollectorInterface;
- this.ksTestIntervalMillis = ksTestIntervalMillis;
- this.ksTrainIntervalMillis = ksTrainIntervalMillis;
- this.hsdevNumHistoricalPeriods = hsdevNumHistoricalPeriods;
-
- this.ksTechnique = new KSTechnique();
- this.hsdevTechnique = new HsdevTechnique();
-
- trendMetrics = new ArrayList<>();
- }
-
- public void runKSTest(long currentEndTime, Set<TrendMetric> trendMetrics) {
- readInputFile(inputFile);
-
- long ksTestIntervalStartTime = currentEndTime - ksTestIntervalMillis;
- LOG.info("Running KS Test for test data interval [" + new Date(ksTestIntervalStartTime) + " : " +
- new Date(currentEndTime) + "], with train data period [" + new Date(ksTestIntervalStartTime - ksTrainIntervalMillis)
- + " : " + new Date(ksTestIntervalStartTime) + "]");
-
- for (TrendMetric metric : trendMetrics) {
- String metricName = metric.metricName;
- String appId = metric.appId;
- String hostname = metric.hostname;
- String key = metricName + ":" + appId + ":" + hostname;
-
- TimelineMetrics ksData = metricsCollectorInterface.fetchMetrics(metricName, appId, hostname, ksTestIntervalStartTime - ksTrainIntervalMillis,
- currentEndTime);
-
- if (ksData.getMetrics().isEmpty()) {
- LOG.info("No metrics fetched for KS, metricKey = " + key);
- continue;
- }
-
- List<Double> trainTsList = new ArrayList<>();
- List<Double> trainDataList = new ArrayList<>();
- List<Double> testTsList = new ArrayList<>();
- List<Double> testDataList = new ArrayList<>();
-
- for (TimelineMetric timelineMetric : ksData.getMetrics()) {
- for (Long timestamp : timelineMetric.getMetricValues().keySet()) {
- if (timestamp <= ksTestIntervalStartTime) {
- trainDataList.add(timelineMetric.getMetricValues().get(timestamp));
- trainTsList.add((double) timestamp);
- } else {
- testDataList.add(timelineMetric.getMetricValues().get(timestamp));
- testTsList.add((double) timestamp);
- }
- }
- }
-
- LOG.info("Train Data size : " + trainDataList.size() + ", Test Data Size : " + testDataList.size());
- if (trainDataList.isEmpty() || testDataList.isEmpty() || trainDataList.size() < testDataList.size()) {
- LOG.info("Not enough train/test data to perform KS analysis.");
- continue;
- }
-
- String ksTrainSeries = "KSTrainSeries";
- double[] trainTs = new double[trainTsList.size()];
- double[] trainData = new double[trainTsList.size()];
- for (int i = 0; i < trainTs.length; i++) {
- trainTs[i] = trainTsList.get(i);
- trainData[i] = trainDataList.get(i);
- }
-
- String ksTestSeries = "KSTestSeries";
- double[] testTs = new double[testTsList.size()];
- double[] testData = new double[testTsList.size()];
- for (int i = 0; i < testTs.length; i++) {
- testTs[i] = testTsList.get(i);
- testData[i] = testDataList.get(i);
- }
-
- LOG.info("Train Size = " + trainTs.length + ", Test Size = " + testTs.length);
-
- DataSeries ksTrainData = new DataSeries(ksTrainSeries, trainTs, trainData);
- DataSeries ksTestData = new DataSeries(ksTestSeries, testTs, testData);
-
- MetricAnomaly metricAnomaly = ksTechnique.runKsTest(key, ksTrainData, ksTestData);
- if (metricAnomaly == null) {
- LOG.info("No anomaly from KS test.");
- } else {
- LOG.info("Found Anomaly in KS Test. Publishing KS Anomaly metric....");
- TimelineMetric timelineMetric = getAsTimelineMetric(metricAnomaly,
- ksTestIntervalStartTime, currentEndTime, ksTestIntervalStartTime - ksTrainIntervalMillis, ksTestIntervalStartTime);
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- timelineMetrics.addOrMergeTimelineMetric(timelineMetric);
- metricsCollectorInterface.emitMetrics(timelineMetrics);
-
- trackedKsAnomalies.put(new KsSingleRunKey(ksTestIntervalStartTime, currentEndTime, metricName, appId, hostname), metricAnomaly);
- }
- }
-
- if (trendMetrics.isEmpty()) {
- LOG.info("No Trend metrics tracked!!!!");
- }
-
- }
-
- private TimelineMetric getAsTimelineMetric(MetricAnomaly metricAnomaly,
- long testStart,
- long testEnd,
- long trainStart,
- long trainEnd) {
-
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName(metricAnomaly.getMetricKey());
- timelineMetric.setAppId(MetricsCollectorInterface.serviceName + "-" + metricAnomaly.getMethodType());
- timelineMetric.setInstanceId(null);
- timelineMetric.setHostName(MetricsCollectorInterface.getDefaultLocalHostName());
- timelineMetric.setStartTime(testEnd);
- HashMap<String, String> metadata = new HashMap<>();
- metadata.put("method", metricAnomaly.getMethodType());
- metadata.put("anomaly-score", String.valueOf(metricAnomaly.getAnomalyScore()));
- metadata.put("test-start-time", String.valueOf(testStart));
- metadata.put("train-start-time", String.valueOf(trainStart));
- metadata.put("train-end-time", String.valueOf(trainEnd));
- timelineMetric.setMetadata(metadata);
- TreeMap<Long,Double> metricValues = new TreeMap<>();
- metricValues.put(testEnd, metricAnomaly.getMetricValue());
- timelineMetric.setMetricValues(metricValues);
- return timelineMetric;
-
- }
-
- public void runHsdevMethod() {
-
- List<TimelineMetric> hsdevMetricAnomalies = new ArrayList<>();
-
- for (KsSingleRunKey ksSingleRunKey : trackedKsAnomalies.keySet()) {
-
- long hsdevTestEnd = ksSingleRunKey.endTime;
- long hsdevTestStart = ksSingleRunKey.startTime;
-
- long period = hsdevTestEnd - hsdevTestStart;
-
- long hsdevTrainStart = hsdevTestStart - (hsdevNumHistoricalPeriods) * period;
- long hsdevTrainEnd = hsdevTestStart;
-
- LOG.info("Running HSdev Test for test data interval [" + new Date(hsdevTestStart) + " : " +
- new Date(hsdevTestEnd) + "], with train data period [" + new Date(hsdevTrainStart)
- + " : " + new Date(hsdevTrainEnd) + "]");
-
- String metricName = ksSingleRunKey.metricName;
- String appId = ksSingleRunKey.appId;
- String hostname = ksSingleRunKey.hostname;
- String key = metricName + "_" + appId + "_" + hostname;
-
- TimelineMetrics hsdevData = metricsCollectorInterface.fetchMetrics(
- metricName,
- appId,
- hostname,
- hsdevTrainStart,
- hsdevTestEnd);
-
- if (hsdevData.getMetrics().isEmpty()) {
- LOG.info("No metrics fetched for HSDev, metricKey = " + key);
- continue;
- }
-
- List<Double> trainTsList = new ArrayList<>();
- List<Double> trainDataList = new ArrayList<>();
- List<Double> testTsList = new ArrayList<>();
- List<Double> testDataList = new ArrayList<>();
-
- for (TimelineMetric timelineMetric : hsdevData.getMetrics()) {
- for (Long timestamp : timelineMetric.getMetricValues().keySet()) {
- if (timestamp <= hsdevTestStart) {
- trainDataList.add(timelineMetric.getMetricValues().get(timestamp));
- trainTsList.add((double) timestamp);
- } else {
- testDataList.add(timelineMetric.getMetricValues().get(timestamp));
- testTsList.add((double) timestamp);
- }
- }
- }
-
- if (trainDataList.isEmpty() || testDataList.isEmpty() || trainDataList.size() < testDataList.size()) {
- LOG.info("Not enough train/test data to perform Hsdev analysis.");
- continue;
- }
-
- String hsdevTrainSeries = "HsdevTrainSeries";
- double[] trainTs = new double[trainTsList.size()];
- double[] trainData = new double[trainTsList.size()];
- for (int i = 0; i < trainTs.length; i++) {
- trainTs[i] = trainTsList.get(i);
- trainData[i] = trainDataList.get(i);
- }
-
- String hsdevTestSeries = "HsdevTestSeries";
- double[] testTs = new double[testTsList.size()];
- double[] testData = new double[testTsList.size()];
- for (int i = 0; i < testTs.length; i++) {
- testTs[i] = testTsList.get(i);
- testData[i] = testDataList.get(i);
- }
-
- LOG.info("Train Size = " + trainTs.length + ", Test Size = " + testTs.length);
-
- DataSeries hsdevTrainData = new DataSeries(hsdevTrainSeries, trainTs, trainData);
- DataSeries hsdevTestData = new DataSeries(hsdevTestSeries, testTs, testData);
-
- MetricAnomaly metricAnomaly = hsdevTechnique.runHsdevTest(key, hsdevTrainData, hsdevTestData);
- if (metricAnomaly == null) {
- LOG.info("No anomaly from Hsdev test. Mismatch between KS and HSDev. ");
- ksTechnique.updateModel(key, false, 10);
- } else {
- LOG.info("Found Anomaly in Hsdev Test. This confirms KS anomaly.");
- hsdevMetricAnomalies.add(getAsTimelineMetric(metricAnomaly,
- hsdevTestStart, hsdevTestEnd, hsdevTrainStart, hsdevTrainEnd));
- }
- }
- clearTrackedKsRunKeys();
-
- if (!hsdevMetricAnomalies.isEmpty()) {
- LOG.info("Publishing Hsdev Anomalies....");
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- timelineMetrics.setMetrics(hsdevMetricAnomalies);
- metricsCollectorInterface.emitMetrics(timelineMetrics);
- }
- }
-
- private void clearTrackedKsRunKeys() {
- trackedKsAnomalies.clear();
- }
-
- private void readInputFile(String fileName) {
- trendMetrics.clear();
- try (BufferedReader br = new BufferedReader(new FileReader(fileName))) {
- for (String line; (line = br.readLine()) != null; ) {
- String[] splits = line.split(",");
- LOG.info("Adding a new metric to track in Trend AD system : " + splits[0]);
- trendMetrics.add(new TrendMetric(splits[0], splits[1], splits[2]));
- }
- } catch (IOException e) {
- LOG.error("Error reading input file : " + e);
- }
- }
-
- class KsSingleRunKey implements Serializable{
-
- long startTime;
- long endTime;
- String metricName;
- String appId;
- String hostname;
-
- public KsSingleRunKey(long startTime, long endTime, String metricName, String appId, String hostname) {
- this.startTime = startTime;
- this.endTime = endTime;
- this.metricName = metricName;
- this.appId = appId;
- this.hostname = hostname;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/TrendMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/TrendMetric.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/TrendMetric.java
deleted file mode 100644
index 0640142..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/TrendMetric.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.core;
-
-import java.io.Serializable;
-
-public class TrendMetric implements Serializable {
-
- String metricName;
- String appId;
- String hostname;
-
- public TrendMetric(String metricName, String appId, String hostname) {
- this.metricName = metricName;
- this.appId = appId;
- this.hostname = hostname;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/AnomalyDetectionTechnique.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/AnomalyDetectionTechnique.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/AnomalyDetectionTechnique.java
deleted file mode 100644
index 0b10b4b..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/AnomalyDetectionTechnique.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.methods;
-
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-
-import java.sql.Time;
-import java.util.List;
-import java.util.Map;
-
-public abstract class AnomalyDetectionTechnique {
-
- protected String methodType;
-
- public abstract List<MetricAnomaly> test(TimelineMetric metric);
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/MetricAnomaly.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/MetricAnomaly.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/MetricAnomaly.java
deleted file mode 100644
index da4f030..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/MetricAnomaly.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.methods;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-
-public class MetricAnomaly implements Serializable{
-
- private String methodType;
- private double anomalyScore;
- private String metricKey;
- private long timestamp;
- private double metricValue;
-
-
- public MetricAnomaly(String metricKey, long timestamp, double metricValue, String methodType, double anomalyScore) {
- this.metricKey = metricKey;
- this.timestamp = timestamp;
- this.metricValue = metricValue;
- this.methodType = methodType;
- this.anomalyScore = anomalyScore;
-
- }
-
- public String getMethodType() {
- return methodType;
- }
-
- public void setMethodType(String methodType) {
- this.methodType = methodType;
- }
-
- public double getAnomalyScore() {
- return anomalyScore;
- }
-
- public void setAnomalyScore(double anomalyScore) {
- this.anomalyScore = anomalyScore;
- }
-
- public void setMetricKey(String metricKey) {
- this.metricKey = metricKey;
- }
-
- public String getMetricKey() {
- return metricKey;
- }
-
- public void setMetricName(String metricName) {
- this.metricKey = metricName;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- }
-
- public double getMetricValue() {
- return metricValue;
- }
-
- public void setMetricValue(double metricValue) {
- this.metricValue = metricValue;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaModel.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaModel.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaModel.java
deleted file mode 100644
index a31410d..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaModel.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.methods.ema;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
-
-import static org.apache.ambari.metrics.alertservice.prototype.methods.ema.EmaTechnique.suppressAnomaliesTheshold;
-
-@XmlRootElement
-public class EmaModel implements Serializable {
-
- private String metricName;
- private String hostname;
- private String appId;
- private double ema;
- private double ems;
- private double weight;
- private double timessdev;
-
- private int ctr = 0;
-
- private static final Log LOG = LogFactory.getLog(EmaModel.class);
-
- public EmaModel(String name, String hostname, String appId, double weight, double timessdev) {
- this.metricName = name;
- this.hostname = hostname;
- this.appId = appId;
- this.weight = weight;
- this.timessdev = timessdev;
- this.ema = 0.0;
- this.ems = 0.0;
- }
-
- public String getMetricName() {
- return metricName;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public String getAppId() {
- return appId;
- }
-
- public double testAndUpdate(double metricValue) {
-
- double anomalyScore = 0.0;
- LOG.info("Before Update ->" + metricName + ":" + appId + ":" + hostname + " - " + "ema = " + ema + ", ems = " + ems + ", timessdev = " + timessdev);
- update(metricValue);
- if (ctr > suppressAnomaliesTheshold) {
- anomalyScore = test(metricValue);
- if (anomalyScore > 0.0) {
- LOG.info("Anomaly ->" + metricName + ":" + appId + ":" + hostname + " - " + "ema = " + ema + ", ems = " + ems +
- ", timessdev = " + timessdev + ", metricValue = " + metricValue);
- } else {
- LOG.info("Not an Anomaly ->" + metricName + ":" + appId + ":" + hostname + " - " + "ema = " + ema + ", ems = " + ems +
- ", timessdev = " + timessdev + ", metricValue = " + metricValue);
- }
- } else {
- ctr++;
- if (ctr > suppressAnomaliesTheshold) {
- LOG.info("Ema Model for " + metricName + ":" + appId + ":" + hostname + " is ready for testing data.");
- }
- }
- return anomalyScore;
- }
-
- public void update(double metricValue) {
- ema = weight * ema + (1 - weight) * metricValue;
- ems = Math.sqrt(weight * Math.pow(ems, 2.0) + (1 - weight) * Math.pow(metricValue - ema, 2.0));
- LOG.debug("In update : ema = " + ema + ", ems = " + ems);
- }
-
- public double test(double metricValue) {
- LOG.debug("In test : ema = " + ema + ", ems = " + ems);
- double diff = Math.abs(ema - metricValue) - (timessdev * ems);
- LOG.debug("diff = " + diff);
- if (diff > 0) {
- return Math.abs((metricValue - ema) / ems); //Z score
- } else {
- return 0.0;
- }
- }
-
- public void updateModel(boolean increaseSensitivity, double percent) {
- LOG.info("Updating model for " + metricName + " with increaseSensitivity = " + increaseSensitivity + ", percent = " + percent);
- double delta = percent / 100;
- if (increaseSensitivity) {
- delta = delta * -1;
- }
- this.timessdev = timessdev + delta * timessdev;
- //this.weight = Math.min(1.0, weight + delta * weight);
- LOG.info("New model parameters " + metricName + " : timessdev = " + timessdev + ", weight = " + weight);
- }
-
- public double getWeight() {
- return weight;
- }
-
- public void setWeight(double weight) {
- this.weight = weight;
- }
-
- public double getTimessdev() {
- return timessdev;
- }
-
- public void setTimessdev(double timessdev) {
- this.timessdev = timessdev;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaModelLoader.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaModelLoader.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaModelLoader.java
deleted file mode 100644
index 62749c1..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaModelLoader.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.methods.ema;
-
-import com.google.gson.Gson;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.spark.SparkContext;
-import org.apache.spark.mllib.util.Loader;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-public class EmaModelLoader implements Loader<EmaTechnique> {
- private static final Log LOG = LogFactory.getLog(EmaModelLoader.class);
-
- @Override
- public EmaTechnique load(SparkContext sc, String path) {
- return new EmaTechnique(0.5,3);
-// Gson gson = new Gson();
-// try {
-// String fileString = new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8);
-// return gson.fromJson(fileString, EmaTechnique.class);
-// } catch (IOException e) {
-// LOG.error(e);
-// }
-// return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaTechnique.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaTechnique.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaTechnique.java
deleted file mode 100644
index 52c6cf3..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/ema/EmaTechnique.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.methods.ema;
-
-import com.google.gson.Gson;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.ambari.metrics.alertservice.prototype.methods.MetricAnomaly;
-import org.apache.ambari.metrics.alertservice.prototype.methods.AnomalyDetectionTechnique;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.spark.SparkContext;
-import org.apache.spark.mllib.util.Saveable;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.BufferedWriter;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.Serializable;
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@XmlRootElement
-public class EmaTechnique extends AnomalyDetectionTechnique implements Serializable, Saveable {
-
- @XmlElement(name = "trackedEmas")
- private Map<String, EmaModel> trackedEmas;
- private static final Log LOG = LogFactory.getLog(EmaTechnique.class);
-
- private double startingWeight = 0.5;
- private double startTimesSdev = 3.0;
- private String methodType = "ema";
- public static int suppressAnomaliesTheshold = 100;
-
- public EmaTechnique(double startingWeight, double startTimesSdev, int suppressAnomaliesTheshold) {
- trackedEmas = new HashMap<>();
- this.startingWeight = startingWeight;
- this.startTimesSdev = startTimesSdev;
- EmaTechnique.suppressAnomaliesTheshold = suppressAnomaliesTheshold;
- LOG.info("New EmaTechnique......");
- }
-
- public EmaTechnique(double startingWeight, double startTimesSdev) {
- trackedEmas = new HashMap<>();
- this.startingWeight = startingWeight;
- this.startTimesSdev = startTimesSdev;
- LOG.info("New EmaTechnique......");
- }
-
- public List<MetricAnomaly> test(TimelineMetric metric) {
- String metricName = metric.getMetricName();
- String appId = metric.getAppId();
- String hostname = metric.getHostName();
- String key = metricName + ":" + appId + ":" + hostname;
-
- EmaModel emaModel = trackedEmas.get(key);
- if (emaModel == null) {
- LOG.debug("EmaModel not present for " + key);
- LOG.debug("Number of tracked Emas : " + trackedEmas.size());
- emaModel = new EmaModel(metricName, hostname, appId, startingWeight, startTimesSdev);
- trackedEmas.put(key, emaModel);
- } else {
- LOG.debug("EmaModel already present for " + key);
- }
-
- List<MetricAnomaly> anomalies = new ArrayList<>();
-
- for (Long timestamp : metric.getMetricValues().keySet()) {
- double metricValue = metric.getMetricValues().get(timestamp);
- double anomalyScore = emaModel.testAndUpdate(metricValue);
- if (anomalyScore > 0.0) {
- LOG.info("Found anomaly for : " + key + ", anomalyScore = " + anomalyScore);
- MetricAnomaly metricAnomaly = new MetricAnomaly(key, timestamp, metricValue, methodType, anomalyScore);
- anomalies.add(metricAnomaly);
- } else {
- LOG.debug("Discarding non-anomaly for : " + key);
- }
- }
- return anomalies;
- }
-
- public boolean updateModel(TimelineMetric timelineMetric, boolean increaseSensitivity, double percent) {
- String metricName = timelineMetric.getMetricName();
- String appId = timelineMetric.getAppId();
- String hostname = timelineMetric.getHostName();
- String key = metricName + "_" + appId + "_" + hostname;
-
-
- EmaModel emaModel = trackedEmas.get(key);
-
- if (emaModel == null) {
- LOG.warn("EMA Model for " + key + " not found");
- return false;
- }
- emaModel.updateModel(increaseSensitivity, percent);
-
- return true;
- }
-
- @Override
- public void save(SparkContext sc, String path) {
- Gson gson = new Gson();
- try {
- String json = gson.toJson(this);
- try (Writer writer = new BufferedWriter(new OutputStreamWriter(
- new FileOutputStream(path), "utf-8"))) {
- writer.write(json);
- }
- } catch (IOException e) {
- LOG.error(e);
- }
- }
-
- @Override
- public String formatVersion() {
- return "1.0";
- }
-
- public Map<String, EmaModel> getTrackedEmas() {
- return trackedEmas;
- }
-
- public double getStartingWeight() {
- return startingWeight;
- }
-
- public double getStartTimesSdev() {
- return startTimesSdev;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/hsdev/HsdevTechnique.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/hsdev/HsdevTechnique.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/hsdev/HsdevTechnique.java
deleted file mode 100644
index 04f4a73..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/hsdev/HsdevTechnique.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.methods.hsdev;
-
-import org.apache.ambari.metrics.alertservice.prototype.common.DataSeries;
-import org.apache.ambari.metrics.alertservice.prototype.methods.MetricAnomaly;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import static org.apache.ambari.metrics.alertservice.prototype.common.StatisticUtils.median;
-import static org.apache.ambari.metrics.alertservice.prototype.common.StatisticUtils.sdev;
-
-import java.io.Serializable;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-
-public class HsdevTechnique implements Serializable {
-
- private Map<String, Double> hsdevMap;
- private String methodType = "hsdev";
- private static final Log LOG = LogFactory.getLog(HsdevTechnique.class);
-
- public HsdevTechnique() {
- hsdevMap = new HashMap<>();
- }
-
- public MetricAnomaly runHsdevTest(String key, DataSeries trainData, DataSeries testData) {
- int testLength = testData.values.length;
- int trainLength = trainData.values.length;
-
- if (trainLength < testLength) {
- LOG.info("Not enough train data.");
- return null;
- }
-
- if (!hsdevMap.containsKey(key)) {
- hsdevMap.put(key, 3.0);
- }
-
- double n = hsdevMap.get(key);
-
- double historicSd = sdev(trainData.values, false);
- double historicMedian = median(trainData.values);
- double currentMedian = median(testData.values);
-
-
- if (historicSd > 0) {
- double diff = Math.abs(currentMedian - historicMedian);
- LOG.info("Found anomaly for metric : " + key + " in the period ending " + new Date((long)testData.ts[testLength - 1]));
- LOG.info("Current median = " + currentMedian + ", Historic Median = " + historicMedian + ", HistoricSd = " + historicSd);
-
- if (diff > n * historicSd) {
- double zScore = diff / historicSd;
- LOG.info("Z Score of current series : " + zScore);
- return new MetricAnomaly(key,
- (long) testData.ts[testLength - 1],
- testData.values[testLength - 1],
- methodType,
- zScore);
- }
- }
-
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/kstest/KSTechnique.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/kstest/KSTechnique.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/kstest/KSTechnique.java
deleted file mode 100644
index a9360d3..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/methods/kstest/KSTechnique.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.alertservice.prototype.methods.kstest;
-
-import org.apache.ambari.metrics.alertservice.prototype.core.RFunctionInvoker;
-import org.apache.ambari.metrics.alertservice.prototype.common.DataSeries;
-import org.apache.ambari.metrics.alertservice.prototype.common.ResultSet;
-import org.apache.ambari.metrics.alertservice.prototype.methods.MetricAnomaly;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-public class KSTechnique implements Serializable {
-
- private String methodType = "ks";
- private Map<String, Double> pValueMap;
- private static final Log LOG = LogFactory.getLog(KSTechnique.class);
-
- public KSTechnique() {
- pValueMap = new HashMap();
- }
-
- public MetricAnomaly runKsTest(String key, DataSeries trainData, DataSeries testData) {
-
- int testLength = testData.values.length;
- int trainLength = trainData.values.length;
-
- if (trainLength < testLength) {
- LOG.info("Not enough train data.");
- return null;
- }
-
- if (!pValueMap.containsKey(key)) {
- pValueMap.put(key, 0.05);
- }
- double pValue = pValueMap.get(key);
-
- ResultSet result = RFunctionInvoker.ksTest(trainData, testData, Collections.singletonMap("ks.p_value", String.valueOf(pValue)));
- if (result == null) {
- LOG.error("Resultset is null when invoking KS R function...");
- return null;
- }
-
- if (result.resultset.size() > 0) {
-
- LOG.info("Is size 1 ? result size = " + result.resultset.get(0).length);
- LOG.info("p_value = " + result.resultset.get(3)[0]);
- double dValue = result.resultset.get(2)[0];
-
- return new MetricAnomaly(key,
- (long) testData.ts[testLength - 1],
- testData.values[testLength - 1],
- methodType,
- dValue);
- }
-
- return null;
- }
-
- public void updateModel(String metricKey, boolean increaseSensitivity, double percent) {
-
- LOG.info("Updating KS model for " + metricKey + " with increaseSensitivity = " + increaseSensitivity + ", percent = " + percent);
-
- if (!pValueMap.containsKey(metricKey)) {
- LOG.error("Unknown metric key : " + metricKey);
- LOG.info("pValueMap :" + pValueMap.toString());
- return;
- }
-
- double delta = percent / 100;
- if (!increaseSensitivity) {
- delta = delta * -1;
- }
-
- double pValue = pValueMap.get(metricKey);
- double newPValue = Math.min(1.0, pValue + delta * pValue);
- pValueMap.put(metricKey, newPValue);
- LOG.info("New pValue = " + newPValue);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyDetectorTestInput.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyDetectorTestInput.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyDetectorTestInput.java
deleted file mode 100644
index 268cd15..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyDetectorTestInput.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.alertservice.prototype.testing.utilities;
-
-import javax.xml.bind.annotation.XmlRootElement;
-import java.util.List;
-import java.util.Map;
-
-@XmlRootElement
-public class MetricAnomalyDetectorTestInput {
-
- public MetricAnomalyDetectorTestInput() {
- }
-
- //Train data
- private String trainDataName;
- private String trainDataType;
- private Map<String, String> trainDataConfigs;
- private int trainDataSize;
-
- //Test data
- private String testDataName;
- private String testDataType;
- private Map<String, String> testDataConfigs;
- private int testDataSize;
-
- //Algorithm data
- private List<String> methods;
- private Map<String, String> methodConfigs;
-
- public String getTrainDataName() {
- return trainDataName;
- }
-
- public void setTrainDataName(String trainDataName) {
- this.trainDataName = trainDataName;
- }
-
- public String getTrainDataType() {
- return trainDataType;
- }
-
- public void setTrainDataType(String trainDataType) {
- this.trainDataType = trainDataType;
- }
-
- public Map<String, String> getTrainDataConfigs() {
- return trainDataConfigs;
- }
-
- public void setTrainDataConfigs(Map<String, String> trainDataConfigs) {
- this.trainDataConfigs = trainDataConfigs;
- }
-
- public String getTestDataName() {
- return testDataName;
- }
-
- public void setTestDataName(String testDataName) {
- this.testDataName = testDataName;
- }
-
- public String getTestDataType() {
- return testDataType;
- }
-
- public void setTestDataType(String testDataType) {
- this.testDataType = testDataType;
- }
-
- public Map<String, String> getTestDataConfigs() {
- return testDataConfigs;
- }
-
- public void setTestDataConfigs(Map<String, String> testDataConfigs) {
- this.testDataConfigs = testDataConfigs;
- }
-
- public Map<String, String> getMethodConfigs() {
- return methodConfigs;
- }
-
- public void setMethodConfigs(Map<String, String> methodConfigs) {
- this.methodConfigs = methodConfigs;
- }
-
- public int getTrainDataSize() {
- return trainDataSize;
- }
-
- public void setTrainDataSize(int trainDataSize) {
- this.trainDataSize = trainDataSize;
- }
-
- public int getTestDataSize() {
- return testDataSize;
- }
-
- public void setTestDataSize(int testDataSize) {
- this.testDataSize = testDataSize;
- }
-
- public List<String> getMethods() {
- return methods;
- }
-
- public void setMethods(List<String> methods) {
- this.methods = methods;
- }
-}
[6/6] ambari git commit: AMBARI-22077 : Create maven module and
package structure for the anomaly detection engine. (Commit 2) (avijayan)
Posted by av...@apache.org.
AMBARI-22077 : Create maven module and package structure for the anomaly detection engine. (Commit 2) (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4613b471
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4613b471
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4613b471
Branch: refs/heads/branch-3.0-ams
Commit: 4613b471e20257df7f1f732e9444d8a90c71d743
Parents: e33b545
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Wed Sep 27 15:02:56 2017 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Wed Sep 27 15:02:56 2017 -0700
----------------------------------------------------------------------
.../pom.xml | 205 ++++++++++
.../adservice/prototype/common/DataSeries.java | 38 ++
.../adservice/prototype/common/ResultSet.java | 43 +++
.../prototype/common/StatisticUtils.java | 62 +++
.../prototype/core/AmbariServerInterface.java | 121 ++++++
.../prototype/core/MetricKafkaProducer.java | 56 +++
.../prototype/core/MetricSparkConsumer.java | 239 ++++++++++++
.../core/MetricsCollectorInterface.java | 237 ++++++++++++
.../prototype/core/PointInTimeADSystem.java | 260 +++++++++++++
.../prototype/core/RFunctionInvoker.java | 222 +++++++++++
.../adservice/prototype/core/TrendADSystem.java | 317 ++++++++++++++++
.../adservice/prototype/core/TrendMetric.java | 33 ++
.../methods/AnomalyDetectionTechnique.java | 30 ++
.../prototype/methods/MetricAnomaly.java | 86 +++++
.../prototype/methods/ema/EmaModel.java | 131 +++++++
.../prototype/methods/ema/EmaModelLoader.java | 40 ++
.../prototype/methods/ema/EmaTechnique.java | 151 ++++++++
.../prototype/methods/hsdev/HsdevTechnique.java | 81 ++++
.../prototype/methods/kstest/KSTechnique.java | 101 +++++
.../MetricAnomalyDetectorTestInput.java | 126 ++++++
.../testing/utilities/MetricAnomalyTester.java | 168 ++++++++
.../utilities/TestMetricSeriesGenerator.java | 92 +++++
.../utilities/TestSeriesInputRequest.java | 88 +++++
.../src/main/resources/R-scripts/ema.R | 96 +++++
.../src/main/resources/R-scripts/hsdev.r | 67 ++++
.../src/main/resources/R-scripts/iforest.R | 52 +++
.../src/main/resources/R-scripts/kstest.r | 38 ++
.../src/main/resources/R-scripts/test.R | 85 +++++
.../src/main/resources/R-scripts/tukeys.r | 51 +++
.../src/main/resources/input-config.properties | 42 ++
.../spark/prototype/MetricAnomalyDetector.scala | 126 ++++++
.../spark/prototype/SparkPhoenixReader.scala | 78 ++++
.../adservice/prototype/TestEmaTechnique.java | 106 ++++++
.../prototype/TestRFunctionInvoker.java | 161 ++++++++
.../metrics/adservice/prototype/TestTukeys.java | 100 +++++
.../seriesgenerator/AbstractMetricSeries.java | 25 ++
.../seriesgenerator/DualBandMetricSeries.java | 88 +++++
.../MetricSeriesGeneratorFactory.java | 377 ++++++++++++++++++
.../MetricSeriesGeneratorTest.java | 101 +++++
.../seriesgenerator/MonotonicMetricSeries.java | 101 +++++
.../seriesgenerator/NormalMetricSeries.java | 81 ++++
.../SteadyWithTurbulenceMetricSeries.java | 115 ++++++
.../StepFunctionMetricSeries.java | 107 ++++++
.../seriesgenerator/UniformMetricSeries.java | 95 +++++
.../ambari-metrics-anomaly-detector/pom.xml | 205 ----------
.../prototype/common/DataSeries.java | 38 --
.../prototype/common/ResultSet.java | 43 ---
.../prototype/common/StatisticUtils.java | 62 ---
.../prototype/core/AmbariServerInterface.java | 121 ------
.../prototype/core/MetricKafkaProducer.java | 56 ---
.../prototype/core/MetricSparkConsumer.java | 239 ------------
.../core/MetricsCollectorInterface.java | 237 ------------
.../prototype/core/PointInTimeADSystem.java | 260 -------------
.../prototype/core/RFunctionInvoker.java | 222 -----------
.../prototype/core/TrendADSystem.java | 317 ----------------
.../prototype/core/TrendMetric.java | 33 --
.../methods/AnomalyDetectionTechnique.java | 32 --
.../prototype/methods/MetricAnomaly.java | 86 -----
.../prototype/methods/ema/EmaModel.java | 131 -------
.../prototype/methods/ema/EmaModelLoader.java | 46 ---
.../prototype/methods/ema/EmaTechnique.java | 151 --------
.../prototype/methods/hsdev/HsdevTechnique.java | 81 ----
.../prototype/methods/kstest/KSTechnique.java | 101 -----
.../MetricAnomalyDetectorTestInput.java | 126 ------
.../testing/utilities/MetricAnomalyTester.java | 166 --------
.../utilities/TestMetricSeriesGenerator.java | 92 -----
.../utilities/TestSeriesInputRequest.java | 88 -----
.../seriesgenerator/AbstractMetricSeries.java | 25 --
.../seriesgenerator/DualBandMetricSeries.java | 88 -----
.../MetricSeriesGeneratorFactory.java | 379 -------------------
.../seriesgenerator/MonotonicMetricSeries.java | 101 -----
.../seriesgenerator/NormalMetricSeries.java | 81 ----
.../SteadyWithTurbulenceMetricSeries.java | 115 ------
.../StepFunctionMetricSeries.java | 107 ------
.../seriesgenerator/UniformMetricSeries.java | 95 -----
.../src/main/resources/R-scripts/ema.R | 96 -----
.../src/main/resources/R-scripts/hsdev.r | 67 ----
.../src/main/resources/R-scripts/iforest.R | 52 ---
.../src/main/resources/R-scripts/kstest.r | 38 --
.../src/main/resources/R-scripts/test.R | 85 -----
.../src/main/resources/R-scripts/tukeys.r | 51 ---
.../src/main/resources/input-config.properties | 42 --
.../metrics/spark/MetricAnomalyDetector.scala | 127 -------
.../metrics/spark/SparkPhoenixReader.scala | 78 ----
.../prototype/TestEmaTechnique.java | 106 ------
.../prototype/TestRFunctionInvoker.java | 161 --------
.../alertservice/prototype/TestTukeys.java | 100 -----
.../MetricSeriesGeneratorTest.java | 101 -----
ambari-metrics/pom.xml | 2 +-
89 files changed, 5020 insertions(+), 5029 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml b/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml
new file mode 100644
index 0000000..1a10f86
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml
@@ -0,0 +1,205 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>ambari-metrics</artifactId>
+ <groupId>org.apache.ambari</groupId>
+ <version>2.0.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>ambari-metrics-anomaly-detection-service</artifactId>
+ <version>2.0.0.0-SNAPSHOT</version>
+ <properties>
+ <scala.version>2.10.4</scala.version>
+ <scala.binary.version>2.11</scala.binary.version>
+ </properties>
+
+ <repositories>
+ <repository>
+ <id>scala-tools.org</id>
+ <name>Scala-Tools Maven2 Repository</name>
+ <url>http://scala-tools.org/repo-releases</url>
+ </repository>
+ </repositories>
+
+ <pluginRepositories>
+ <pluginRepository>
+ <id>scala-tools.org</id>
+ <name>Scala-Tools Maven2 Repository</name>
+ <url>http://scala-tools.org/repo-releases</url>
+ </pluginRepository>
+ </pluginRepositories>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.scala-tools</groupId>
+ <artifactId>maven-scala-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <scalaVersion>${scala.version}</scalaVersion>
+ <args>
+ <arg>-target:jvm-1.5</arg>
+ </args>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <name>Ambari Metrics Anomaly Detection Service</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.5</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>1.7.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.github.lucarosellini.rJava</groupId>
+ <artifactId>JRI</artifactId>
+ <version>0.9-7</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_2.11</artifactId>
+ <version>2.1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ <version>0.10.1.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jmx</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>0.10.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>connect-json</artifactId>
+ <version>0.10.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming-kafka_2.10</artifactId>
+ <version>1.6.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.10</artifactId>
+ <version>1.6.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-spark</artifactId>
+ <version>4.10.0-HBase-1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-mllib_2.10</artifactId>
+ <version>1.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ <version>4.10</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-metrics-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>2.1.1</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-mllib_${scala.binary.version}</artifactId>
+ <version>2.1.1</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/DataSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/DataSeries.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/DataSeries.java
new file mode 100644
index 0000000..54b402f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/DataSeries.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.common;
+
+import java.util.Arrays;
+
+public class DataSeries {
+
+ public String seriesName;
+ public double[] ts;
+ public double[] values;
+
+ public DataSeries(String seriesName, double[] ts, double[] values) {
+ this.seriesName = seriesName;
+ this.ts = ts;
+ this.values = values;
+ }
+
+ @Override
+ public String toString() {
+ return seriesName + Arrays.toString(ts) + Arrays.toString(values);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/ResultSet.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/ResultSet.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/ResultSet.java
new file mode 100644
index 0000000..dd3038f
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/ResultSet.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.common;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ResultSet {
+
+ public List<double[]> resultset = new ArrayList<>();
+
+ public ResultSet(List<double[]> resultset) {
+ this.resultset = resultset;
+ }
+
+ public void print() {
+ System.out.println("Result : ");
+ if (!resultset.isEmpty()) {
+ for (int i = 0; i<resultset.get(0).length;i++) {
+ for (double[] entity : resultset) {
+ System.out.print(entity[i] + " ");
+ }
+ System.out.println();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/StatisticUtils.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/StatisticUtils.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/StatisticUtils.java
new file mode 100644
index 0000000..7f0aed3
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/StatisticUtils.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.common;
+
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+public class StatisticUtils {
+
+ public static double mean(double[] values) {
+ double sum = 0;
+ for (double d : values) {
+ sum += d;
+ }
+ return sum / values.length;
+ }
+
+ public static double variance(double[] values) {
+ double avg = mean(values);
+ double variance = 0;
+ for (double d : values) {
+ variance += Math.pow(d - avg, 2.0);
+ }
+ return variance;
+ }
+
+ public static double sdev(double[] values, boolean useBesselsCorrection) {
+ double variance = variance(values);
+ int n = (useBesselsCorrection) ? values.length - 1 : values.length;
+ return Math.sqrt(variance / n);
+ }
+
+ public static double median(double[] values) {
+ double[] clonedValues = Arrays.copyOf(values, values.length);
+ Arrays.sort(clonedValues);
+ int n = values.length;
+
+ if (n % 2 != 0) {
+ return clonedValues[(n-1)/2];
+ } else {
+ return ( clonedValues[(n-1)/2] + clonedValues[n/2] ) / 2;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/AmbariServerInterface.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/AmbariServerInterface.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/AmbariServerInterface.java
new file mode 100644
index 0000000..920d758
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/AmbariServerInterface.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.adservice.prototype.core;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class AmbariServerInterface implements Serializable{
+
+ private static final Log LOG = LogFactory.getLog(AmbariServerInterface.class);
+
+ private String ambariServerHost;
+ private String clusterName;
+
+ public AmbariServerInterface(String ambariServerHost, String clusterName) {
+ this.ambariServerHost = ambariServerHost;
+ this.clusterName = clusterName;
+ }
+
+ public int getPointInTimeSensitivity() {
+
+ String url = constructUri("http", ambariServerHost, "8080", "/api/v1/clusters/" + clusterName + "/alert_definitions?fields=*");
+
+ URL obj = null;
+ BufferedReader in = null;
+
+ try {
+ obj = new URL(url);
+ HttpURLConnection con = (HttpURLConnection) obj.openConnection();
+ con.setRequestMethod("GET");
+
+ String encoded = Base64.getEncoder().encodeToString(("admin:admin").getBytes(StandardCharsets.UTF_8));
+ con.setRequestProperty("Authorization", "Basic "+encoded);
+
+ int responseCode = con.getResponseCode();
+ LOG.info("Sending 'GET' request to URL : " + url);
+ LOG.info("Response Code : " + responseCode);
+
+ in = new BufferedReader(
+ new InputStreamReader(con.getInputStream()));
+
+ StringBuilder responseJsonSb = new StringBuilder();
+ String line;
+ while ((line = in.readLine()) != null) {
+ responseJsonSb.append(line);
+ }
+
+ JSONObject jsonObject = new JSONObject(responseJsonSb.toString());
+ JSONArray array = jsonObject.getJSONArray("items");
+ for(int i = 0 ; i < array.length() ; i++){
+ JSONObject alertDefn = array.getJSONObject(i).getJSONObject("AlertDefinition");
+ if (alertDefn.get("name") != null && alertDefn.get("name").equals("point_in_time_metrics_anomalies")) {
+ JSONObject sourceNode = alertDefn.getJSONObject("source");
+ JSONArray params = sourceNode.getJSONArray("parameters");
+ for(int j = 0 ; j < params.length() ; j++){
+ JSONObject param = params.getJSONObject(j);
+ if (param.get("name").equals("sensitivity")) {
+ return param.getInt("value");
+ }
+ }
+ break;
+ }
+ }
+
+ } catch (Exception e) {
+ LOG.error(e);
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ LOG.warn(e);
+ }
+ }
+ }
+
+ return -1;
+ }
+
+ private String constructUri(String protocol, String host, String port, String path) {
+ StringBuilder sb = new StringBuilder(protocol);
+ sb.append("://");
+ sb.append(host);
+ sb.append(":");
+ sb.append(port);
+ sb.append(path);
+ return sb.toString();
+ }
+
+// public static void main(String[] args) {
+// AmbariServerInterface ambariServerInterface = new AmbariServerInterface();
+// ambariServerInterface.getPointInTimeSensitivity("avijayan-ams-1.openstacklocal","c1");
+// }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricKafkaProducer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricKafkaProducer.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricKafkaProducer.java
new file mode 100644
index 0000000..167fbb3
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricKafkaProducer.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.core;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+public class MetricKafkaProducer {
+
+ Producer producer;
+ private static String topicName = "ambari-metrics-topic";
+
+ public MetricKafkaProducer(String kafkaServers) {
+ Properties configProperties = new Properties();
+ configProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServers); //"avijayan-ams-2.openstacklocal:6667"
+ configProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.ByteArraySerializer");
+ configProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.connect.json.JsonSerializer");
+ producer = new KafkaProducer(configProperties);
+ }
+
+ public void sendMetrics(TimelineMetrics timelineMetrics) throws InterruptedException, ExecutionException {
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ JsonNode jsonNode = objectMapper.valueToTree(timelineMetrics);
+ ProducerRecord<String, JsonNode> rec = new ProducerRecord<String, JsonNode>(topicName,jsonNode);
+ Future<RecordMetadata> kafkaFuture = producer.send(rec);
+
+ System.out.println(kafkaFuture.isDone());
+ System.out.println(kafkaFuture.get().topic());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricSparkConsumer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricSparkConsumer.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricSparkConsumer.java
new file mode 100644
index 0000000..e8257e5
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricSparkConsumer.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.core;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
+import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaTechnique;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.spark.streaming.Duration;
+import org.apache.spark.streaming.api.java.JavaDStream;
+import org.apache.spark.streaming.api.java.JavaPairDStream;
+import org.apache.spark.streaming.api.java.JavaPairReceiverInputDStream;
+import org.apache.spark.streaming.api.java.JavaStreamingContext;
+import org.apache.spark.streaming.kafka.KafkaUtils;
+import scala.Tuple2;
+
+import java.util.*;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class MetricSparkConsumer {
+
+ private static final Log LOG = LogFactory.getLog(MetricSparkConsumer.class);
+ private static String groupId = "ambari-metrics-group";
+ private static String topicName = "ambari-metrics-topic";
+ private static int numThreads = 1;
+ private static long pitStartTime = System.currentTimeMillis();
+ private static long ksStartTime = pitStartTime;
+ private static long hdevStartTime = ksStartTime;
+ private static Set<Pattern> includeMetricPatterns = new HashSet<>();
+ private static Set<String> includedHosts = new HashSet<>();
+ private static Set<TrendMetric> trendMetrics = new HashSet<>();
+
+ public MetricSparkConsumer() {
+ }
+
+ public static Properties readProperties(String propertiesFile) {
+ try {
+ Properties properties = new Properties();
+ InputStream inputStream = ClassLoader.getSystemResourceAsStream(propertiesFile);
+ if (inputStream == null) {
+ inputStream = new FileInputStream(propertiesFile);
+ }
+ properties.load(inputStream);
+ return properties;
+ } catch (IOException ioEx) {
+ LOG.error("Error reading properties file for jmeter");
+ return null;
+ }
+ }
+
+ public static void main(String[] args) throws InterruptedException {
+
+ if (args.length < 1) {
+ System.err.println("Usage: MetricSparkConsumer <input-config-file>");
+ System.exit(1);
+ }
+
+ Properties properties = readProperties(args[0]);
+
+ List<String> appIds = Arrays.asList(properties.getProperty("appIds").split(","));
+
+ String collectorHost = properties.getProperty("collectorHost");
+ String collectorPort = properties.getProperty("collectorPort");
+ String collectorProtocol = properties.getProperty("collectorProtocol");
+
+ String zkQuorum = properties.getProperty("zkQuorum");
+
+ double emaW = Double.parseDouble(properties.getProperty("emaW"));
+ double emaN = Double.parseDouble(properties.getProperty("emaN"));
+ int emaThreshold = Integer.parseInt(properties.getProperty("emaThreshold"));
+ double tukeysN = Double.parseDouble(properties.getProperty("tukeysN"));
+
+ long pitTestInterval = Long.parseLong(properties.getProperty("pointInTimeTestInterval"));
+ long pitTrainInterval = Long.parseLong(properties.getProperty("pointInTimeTrainInterval"));
+
+ long ksTestInterval = Long.parseLong(properties.getProperty("ksTestInterval"));
+ long ksTrainInterval = Long.parseLong(properties.getProperty("ksTrainInterval"));
+ int hsdevNhp = Integer.parseInt(properties.getProperty("hsdevNhp"));
+ long hsdevInterval = Long.parseLong(properties.getProperty("hsdevInterval"));
+
+ String ambariServerHost = properties.getProperty("ambariServerHost");
+ String clusterName = properties.getProperty("clusterName");
+
+ String includeMetricPatternStrings = properties.getProperty("includeMetricPatterns");
+ if (includeMetricPatternStrings != null && !includeMetricPatternStrings.isEmpty()) {
+ String[] patterns = includeMetricPatternStrings.split(",");
+ for (String p : patterns) {
+ LOG.info("Included Pattern : " + p);
+ includeMetricPatterns.add(Pattern.compile(p));
+ }
+ }
+
+ String includedHostList = properties.getProperty("hosts");
+ if (includedHostList != null && !includedHostList.isEmpty()) {
+ String[] hosts = includedHostList.split(",");
+ includedHosts.addAll(Arrays.asList(hosts));
+ }
+
+ MetricsCollectorInterface metricsCollectorInterface = new MetricsCollectorInterface(collectorHost, collectorProtocol, collectorPort);
+
+ SparkConf sparkConf = new SparkConf().setAppName("AmbariMetricsAnomalyDetector");
+
+ JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, new Duration(10000));
+
+ EmaTechnique emaTechnique = new EmaTechnique(emaW, emaN, emaThreshold);
+ PointInTimeADSystem pointInTimeADSystem = new PointInTimeADSystem(metricsCollectorInterface,
+ tukeysN,
+ pitTestInterval,
+ pitTrainInterval,
+ ambariServerHost,
+ clusterName);
+
+ TrendADSystem trendADSystem = new TrendADSystem(metricsCollectorInterface,
+ ksTestInterval,
+ ksTrainInterval,
+ hsdevNhp);
+
+ Broadcast<EmaTechnique> emaTechniqueBroadcast = jssc.sparkContext().broadcast(emaTechnique);
+ Broadcast<PointInTimeADSystem> pointInTimeADSystemBroadcast = jssc.sparkContext().broadcast(pointInTimeADSystem);
+ Broadcast<TrendADSystem> trendADSystemBroadcast = jssc.sparkContext().broadcast(trendADSystem);
+ Broadcast<MetricsCollectorInterface> metricsCollectorInterfaceBroadcast = jssc.sparkContext().broadcast(metricsCollectorInterface);
+ Broadcast<Set<Pattern>> includePatternBroadcast = jssc.sparkContext().broadcast(includeMetricPatterns);
+ Broadcast<Set<String>> includedHostBroadcast = jssc.sparkContext().broadcast(includedHosts);
+
+ JavaPairReceiverInputDStream<String, String> messages =
+ KafkaUtils.createStream(jssc, zkQuorum, groupId, Collections.singletonMap(topicName, numThreads));
+
+ //Convert JSON string to TimelineMetrics.
+ JavaDStream<TimelineMetrics> timelineMetricsStream = messages.map(new Function<Tuple2<String, String>, TimelineMetrics>() {
+ @Override
+ public TimelineMetrics call(Tuple2<String, String> message) throws Exception {
+ ObjectMapper mapper = new ObjectMapper();
+ TimelineMetrics metrics = mapper.readValue(message._2, TimelineMetrics.class);
+ return metrics;
+ }
+ });
+
+ timelineMetricsStream.print();
+
+ //Group TimelineMetric by AppId.
+ JavaPairDStream<String, TimelineMetrics> appMetricStream = timelineMetricsStream.mapToPair(
+ timelineMetrics -> timelineMetrics.getMetrics().isEmpty() ? new Tuple2<>("TEST", new TimelineMetrics()) : new Tuple2<String, TimelineMetrics>(timelineMetrics.getMetrics().get(0).getAppId(), timelineMetrics)
+ );
+
+ appMetricStream.print();
+
+ //Filter AppIds that are not needed.
+ JavaPairDStream<String, TimelineMetrics> filteredAppMetricStream = appMetricStream.filter(new Function<Tuple2<String, TimelineMetrics>, Boolean>() {
+ @Override
+ public Boolean call(Tuple2<String, TimelineMetrics> appMetricTuple) throws Exception {
+ return appIds.contains(appMetricTuple._1);
+ }
+ });
+
+ filteredAppMetricStream.print();
+
+ filteredAppMetricStream.foreachRDD(rdd -> {
+ rdd.foreach(
+ tuple2 -> {
+ long currentTime = System.currentTimeMillis();
+ EmaTechnique ema = emaTechniqueBroadcast.getValue();
+ if (currentTime > pitStartTime + pitTestInterval) {
+ LOG.info("Running Tukeys....");
+ pointInTimeADSystemBroadcast.getValue().runTukeysAndRefineEma(ema, currentTime);
+ pitStartTime = pitStartTime + pitTestInterval;
+ }
+
+ if (currentTime > ksStartTime + ksTestInterval) {
+ LOG.info("Running KS Test....");
+ trendADSystemBroadcast.getValue().runKSTest(currentTime, trendMetrics);
+ ksStartTime = ksStartTime + ksTestInterval;
+ }
+
+ if (currentTime > hdevStartTime + hsdevInterval) {
+ LOG.info("Running HSdev Test....");
+ trendADSystemBroadcast.getValue().runHsdevMethod();
+ hdevStartTime = hdevStartTime + hsdevInterval;
+ }
+
+ TimelineMetrics metrics = tuple2._2();
+ for (TimelineMetric timelineMetric : metrics.getMetrics()) {
+
+ boolean includeHost = includedHostBroadcast.getValue().contains(timelineMetric.getHostName());
+ boolean includeMetric = false;
+ if (includeHost) {
+ if (includePatternBroadcast.getValue().isEmpty()) {
+ includeMetric = true;
+ }
+ for (Pattern p : includePatternBroadcast.getValue()) {
+ Matcher m = p.matcher(timelineMetric.getMetricName());
+ if (m.find()) {
+ includeMetric = true;
+ }
+ }
+ }
+
+ if (includeMetric) {
+ trendMetrics.add(new TrendMetric(timelineMetric.getMetricName(), timelineMetric.getAppId(),
+ timelineMetric.getHostName()));
+ List<MetricAnomaly> anomalies = ema.test(timelineMetric);
+ metricsCollectorInterfaceBroadcast.getValue().publish(anomalies);
+ }
+ }
+ });
+ });
+
+ jssc.start();
+ jssc.awaitTermination();
+ }
+}
+
+
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricsCollectorInterface.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricsCollectorInterface.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricsCollectorInterface.java
new file mode 100644
index 0000000..da3999a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricsCollectorInterface.java
@@ -0,0 +1,237 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.core;
+
+import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.codehaus.jackson.map.AnnotationIntrospector;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.net.HttpURLConnection;
+import java.net.InetAddress;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.TreeMap;
+
+public class MetricsCollectorInterface implements Serializable {
+
+ private static String hostName = null;
+ private String instanceId = null;
+ public final static String serviceName = "anomaly-engine";
+ private String collectorHost;
+ private String protocol;
+ private String port;
+ private static final String WS_V1_TIMELINE_METRICS = "/ws/v1/timeline/metrics";
+ private static final Log LOG = LogFactory.getLog(MetricsCollectorInterface.class);
+ private static ObjectMapper mapper;
+ private final static ObjectReader timelineObjectReader;
+
+ static {
+ mapper = new ObjectMapper();
+ AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
+ mapper.setAnnotationIntrospector(introspector);
+ mapper.getSerializationConfig()
+ .withSerializationInclusion(JsonSerialize.Inclusion.NON_NULL);
+ timelineObjectReader = mapper.reader(TimelineMetrics.class);
+ }
+
+ public MetricsCollectorInterface(String collectorHost, String protocol, String port) {
+ this.collectorHost = collectorHost;
+ this.protocol = protocol;
+ this.port = port;
+ this.hostName = getDefaultLocalHostName();
+ }
+
+ public static String getDefaultLocalHostName() {
+
+ if (hostName != null) {
+ return hostName;
+ }
+
+ try {
+ return InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ LOG.info("Error getting host address");
+ }
+ return null;
+ }
+
+ public void publish(List<MetricAnomaly> metricAnomalies) {
+ if (CollectionUtils.isNotEmpty(metricAnomalies)) {
+ LOG.info("Sending metric anomalies of size : " + metricAnomalies.size());
+ List<TimelineMetric> metricList = getTimelineMetricList(metricAnomalies);
+ if (!metricList.isEmpty()) {
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+ timelineMetrics.setMetrics(metricList);
+ emitMetrics(timelineMetrics);
+ }
+ } else {
+ LOG.debug("No anomalies to send.");
+ }
+ }
+
+ private List<TimelineMetric> getTimelineMetricList(List<MetricAnomaly> metricAnomalies) {
+ List<TimelineMetric> metrics = new ArrayList<>();
+
+ if (metricAnomalies.isEmpty()) {
+ return metrics;
+ }
+
+ for (MetricAnomaly anomaly : metricAnomalies) {
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setMetricName(anomaly.getMetricKey());
+ timelineMetric.setAppId(serviceName + "-" + anomaly.getMethodType());
+ timelineMetric.setInstanceId(null);
+ timelineMetric.setHostName(getDefaultLocalHostName());
+ timelineMetric.setStartTime(anomaly.getTimestamp());
+ HashMap<String, String> metadata = new HashMap<>();
+ metadata.put("method", anomaly.getMethodType());
+ metadata.put("anomaly-score", String.valueOf(anomaly.getAnomalyScore()));
+ timelineMetric.setMetadata(metadata);
+ TreeMap<Long,Double> metricValues = new TreeMap<>();
+ metricValues.put(anomaly.getTimestamp(), anomaly.getMetricValue());
+ timelineMetric.setMetricValues(metricValues);
+
+ metrics.add(timelineMetric);
+ }
+ return metrics;
+ }
+
+ public boolean emitMetrics(TimelineMetrics metrics) {
+ String connectUrl = constructTimelineMetricUri();
+ String jsonData = null;
+ LOG.debug("EmitMetrics connectUrl = " + connectUrl);
+ try {
+ jsonData = mapper.writeValueAsString(metrics);
+ LOG.info(jsonData);
+ } catch (IOException e) {
+ LOG.error("Unable to parse metrics", e);
+ }
+ if (jsonData != null) {
+ return emitMetricsJson(connectUrl, jsonData);
+ }
+ return false;
+ }
+
+ private HttpURLConnection getConnection(String spec) throws IOException {
+ return (HttpURLConnection) new URL(spec).openConnection();
+ }
+
+ private boolean emitMetricsJson(String connectUrl, String jsonData) {
+ int timeout = 10000;
+ HttpURLConnection connection = null;
+ try {
+ if (connectUrl == null) {
+ throw new IOException("Unknown URL. Unable to connect to metrics collector.");
+ }
+ connection = getConnection(connectUrl);
+
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestProperty("Connection", "Keep-Alive");
+ connection.setConnectTimeout(timeout);
+ connection.setReadTimeout(timeout);
+ connection.setDoOutput(true);
+
+ if (jsonData != null) {
+ try (OutputStream os = connection.getOutputStream()) {
+ os.write(jsonData.getBytes("UTF-8"));
+ }
+ }
+
+ int statusCode = connection.getResponseCode();
+
+ if (statusCode != 200) {
+ LOG.info("Unable to POST metrics to collector, " + connectUrl + ", " +
+ "statusCode = " + statusCode);
+ } else {
+ LOG.info("Metrics posted to Collector " + connectUrl);
+ }
+ return true;
+ } catch (IOException ioe) {
+ LOG.error(ioe.getMessage());
+ }
+ return false;
+ }
+
+ private String constructTimelineMetricUri() {
+ StringBuilder sb = new StringBuilder(protocol);
+ sb.append("://");
+ sb.append(collectorHost);
+ sb.append(":");
+ sb.append(port);
+ sb.append(WS_V1_TIMELINE_METRICS);
+ return sb.toString();
+ }
+
+ public TimelineMetrics fetchMetrics(String metricName,
+ String appId,
+ String hostname,
+ long startime,
+ long endtime) {
+
+ String url = constructTimelineMetricUri() + "?metricNames=" + metricName + "&appId=" + appId +
+ "&hostname=" + hostname + "&startTime=" + startime + "&endTime=" + endtime;
+ LOG.debug("Fetch metrics URL : " + url);
+
+ URL obj = null;
+ BufferedReader in = null;
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+
+ try {
+ obj = new URL(url);
+ HttpURLConnection con = (HttpURLConnection) obj.openConnection();
+ con.setRequestMethod("GET");
+ int responseCode = con.getResponseCode();
+ LOG.debug("Sending 'GET' request to URL : " + url);
+ LOG.debug("Response Code : " + responseCode);
+
+ in = new BufferedReader(
+ new InputStreamReader(con.getInputStream()));
+ timelineMetrics = timelineObjectReader.readValue(in);
+ } catch (Exception e) {
+ LOG.error(e);
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ LOG.warn(e);
+ }
+ }
+ }
+
+ LOG.info("Fetched " + timelineMetrics.getMetrics().size() + " metrics.");
+ return timelineMetrics;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/PointInTimeADSystem.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/PointInTimeADSystem.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/PointInTimeADSystem.java
new file mode 100644
index 0000000..0a2271a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/PointInTimeADSystem.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.core;
+
+import org.apache.ambari.metrics.adservice.prototype.common.ResultSet;
+import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
+import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaModel;
+import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaTechnique;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+public class PointInTimeADSystem implements Serializable {
+
+ //private EmaTechnique emaTechnique;
+ private MetricsCollectorInterface metricsCollectorInterface;
+ private Map<String, Double> tukeysNMap;
+ private double defaultTukeysN = 3;
+
+ private long testIntervalMillis = 5*60*1000; //10mins
+ private long trainIntervalMillis = 15*60*1000; //1hour
+
+ private static final Log LOG = LogFactory.getLog(PointInTimeADSystem.class);
+
+ private AmbariServerInterface ambariServerInterface;
+ private int sensitivity = 50;
+ private int minSensitivity = 0;
+ private int maxSensitivity = 100;
+
+ public PointInTimeADSystem(MetricsCollectorInterface metricsCollectorInterface, double defaultTukeysN,
+ long testIntervalMillis, long trainIntervalMillis, String ambariServerHost, String clusterName) {
+ this.metricsCollectorInterface = metricsCollectorInterface;
+ this.defaultTukeysN = defaultTukeysN;
+ this.tukeysNMap = new HashMap<>();
+ this.testIntervalMillis = testIntervalMillis;
+ this.trainIntervalMillis = trainIntervalMillis;
+ this.ambariServerInterface = new AmbariServerInterface(ambariServerHost, clusterName);
+ LOG.info("Starting PointInTimeADSystem...");
+ }
+
+ public void runTukeysAndRefineEma(EmaTechnique emaTechnique, long startTime) {
+ LOG.info("Running Tukeys for test data interval [" + new Date(startTime - testIntervalMillis) + " : " + new Date(startTime) + "], with train data period [" + new Date(startTime - testIntervalMillis - trainIntervalMillis) + " : " + new Date(startTime - testIntervalMillis) + "]");
+
+ int requiredSensivity = ambariServerInterface.getPointInTimeSensitivity();
+ if (requiredSensivity == -1 || requiredSensivity == sensitivity) {
+ LOG.info("No change in sensitivity needed.");
+ } else {
+ LOG.info("Current tukey's N value = " + defaultTukeysN);
+ if (requiredSensivity > sensitivity) {
+ int targetSensitivity = Math.min(maxSensitivity, requiredSensivity);
+ while (sensitivity < targetSensitivity) {
+ defaultTukeysN = defaultTukeysN + defaultTukeysN * 0.05;
+ sensitivity++;
+ }
+ } else {
+ int targetSensitivity = Math.max(minSensitivity, requiredSensivity);
+ while (sensitivity > targetSensitivity) {
+ defaultTukeysN = defaultTukeysN - defaultTukeysN * 0.05;
+ sensitivity--;
+ }
+ }
+ LOG.info("New tukey's N value = " + defaultTukeysN);
+ }
+
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+ for (String metricKey : emaTechnique.getTrackedEmas().keySet()) {
+ LOG.info("EMA key = " + metricKey);
+ EmaModel emaModel = emaTechnique.getTrackedEmas().get(metricKey);
+ String metricName = emaModel.getMetricName();
+ String appId = emaModel.getAppId();
+ String hostname = emaModel.getHostname();
+
+ TimelineMetrics tukeysData = metricsCollectorInterface.fetchMetrics(metricName, appId, hostname, startTime - (testIntervalMillis + trainIntervalMillis),
+ startTime);
+
+ if (tukeysData.getMetrics().isEmpty()) {
+ LOG.info("No metrics fetched for Tukeys, metricKey = " + metricKey);
+ continue;
+ }
+
+ List<Double> trainTsList = new ArrayList<>();
+ List<Double> trainDataList = new ArrayList<>();
+ List<Double> testTsList = new ArrayList<>();
+ List<Double> testDataList = new ArrayList<>();
+
+ for (TimelineMetric metric : tukeysData.getMetrics()) {
+ for (Long timestamp : metric.getMetricValues().keySet()) {
+ if (timestamp <= (startTime - testIntervalMillis)) {
+ trainDataList.add(metric.getMetricValues().get(timestamp));
+ trainTsList.add((double)timestamp);
+ } else {
+ testDataList.add(metric.getMetricValues().get(timestamp));
+ testTsList.add((double)timestamp);
+ }
+ }
+ }
+
+ if (trainDataList.isEmpty() || testDataList.isEmpty() || trainDataList.size() < testDataList.size()) {
+ LOG.info("Not enough train/test data to perform analysis.");
+ continue;
+ }
+
+ String tukeysTrainSeries = "tukeysTrainSeries";
+ double[] trainTs = new double[trainTsList.size()];
+ double[] trainData = new double[trainTsList.size()];
+ for (int i = 0; i < trainTs.length; i++) {
+ trainTs[i] = trainTsList.get(i);
+ trainData[i] = trainDataList.get(i);
+ }
+
+ String tukeysTestSeries = "tukeysTestSeries";
+ double[] testTs = new double[testTsList.size()];
+ double[] testData = new double[testTsList.size()];
+ for (int i = 0; i < testTs.length; i++) {
+ testTs[i] = testTsList.get(i);
+ testData[i] = testDataList.get(i);
+ }
+
+ LOG.info("Train Size = " + trainTs.length + ", Test Size = " + testTs.length);
+
+ DataSeries tukeysTrainData = new DataSeries(tukeysTrainSeries, trainTs, trainData);
+ DataSeries tukeysTestData = new DataSeries(tukeysTestSeries, testTs, testData);
+
+ if (!tukeysNMap.containsKey(metricKey)) {
+ tukeysNMap.put(metricKey, defaultTukeysN);
+ }
+
+ Map<String, String> configs = new HashMap<>();
+ configs.put("tukeys.n", String.valueOf(tukeysNMap.get(metricKey)));
+
+ ResultSet rs = RFunctionInvoker.tukeys(tukeysTrainData, tukeysTestData, configs);
+
+ List<TimelineMetric> tukeysMetrics = getAsTimelineMetric(rs, metricName, appId, hostname);
+ LOG.info("Tukeys anomalies size : " + tukeysMetrics.size());
+ TreeMap<Long, Double> tukeysMetricValues = new TreeMap<>();
+
+ for (TimelineMetric tukeysMetric : tukeysMetrics) {
+ tukeysMetricValues.putAll(tukeysMetric.getMetricValues());
+ timelineMetrics.addOrMergeTimelineMetric(tukeysMetric);
+ }
+
+ TimelineMetrics emaData = metricsCollectorInterface.fetchMetrics(metricKey, MetricsCollectorInterface.serviceName+"-ema", MetricsCollectorInterface.getDefaultLocalHostName(), startTime - testIntervalMillis, startTime);
+ TreeMap<Long, Double> emaMetricValues = new TreeMap();
+ if (!emaData.getMetrics().isEmpty()) {
+ emaMetricValues = emaData.getMetrics().get(0).getMetricValues();
+ }
+
+ LOG.info("Ema anomalies size : " + emaMetricValues.size());
+ int tp = 0;
+ int tn = 0;
+ int fp = 0;
+ int fn = 0;
+
+ for (double ts : testTs) {
+ long timestamp = (long) ts;
+ if (tukeysMetricValues.containsKey(timestamp)) {
+ if (emaMetricValues.containsKey(timestamp)) {
+ tp++;
+ } else {
+ fn++;
+ }
+ } else {
+ if (emaMetricValues.containsKey(timestamp)) {
+ fp++;
+ } else {
+ tn++;
+ }
+ }
+ }
+
+ double recall = (double) tp / (double) (tp + fn);
+ double precision = (double) tp / (double) (tp + fp);
+ LOG.info("----------------------------");
+ LOG.info("Precision Recall values for " + metricKey);
+ LOG.info("tp=" + tp + ", fp=" + fp + ", tn=" + tn + ", fn=" + fn);
+ LOG.info("----------------------------");
+
+ if (recall < 0.5) {
+ LOG.info("Increasing EMA sensitivity by 10%");
+ emaModel.updateModel(true, 5);
+ } else if (precision < 0.5) {
+ LOG.info("Decreasing EMA sensitivity by 10%");
+ emaModel.updateModel(false, 5);
+ }
+
+ }
+
+ if (emaTechnique.getTrackedEmas().isEmpty()){
+ LOG.info("No EMA Technique keys tracked!!!!");
+ }
+
+ if (!timelineMetrics.getMetrics().isEmpty()) {
+ metricsCollectorInterface.emitMetrics(timelineMetrics);
+ }
+ }
+
+ private static List<TimelineMetric> getAsTimelineMetric(ResultSet result, String metricName, String appId, String hostname) {
+
+ List<TimelineMetric> timelineMetrics = new ArrayList<>();
+
+ if (result == null) {
+ LOG.info("ResultSet from R call is null!!");
+ return null;
+ }
+
+ if (result.resultset.size() > 0) {
+ double[] ts = result.resultset.get(0);
+ double[] metrics = result.resultset.get(1);
+ double[] anomalyScore = result.resultset.get(2);
+ for (int i = 0; i < ts.length; i++) {
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setMetricName(metricName + ":" + appId + ":" + hostname);
+ timelineMetric.setHostName(MetricsCollectorInterface.getDefaultLocalHostName());
+ timelineMetric.setAppId(MetricsCollectorInterface.serviceName + "-tukeys");
+ timelineMetric.setInstanceId(null);
+ timelineMetric.setStartTime((long) ts[i]);
+ TreeMap<Long, Double> metricValues = new TreeMap<>();
+ metricValues.put((long) ts[i], metrics[i]);
+
+ HashMap<String, String> metadata = new HashMap<>();
+ metadata.put("method", "tukeys");
+ if (String.valueOf(anomalyScore[i]).equals("infinity")) {
+ LOG.info("Got anomalyScore = infinity for " + metricName + ":" + appId + ":" + hostname);
+ } else {
+ metadata.put("anomaly-score", String.valueOf(anomalyScore[i]));
+ }
+ timelineMetric.setMetadata(metadata);
+
+ timelineMetric.setMetricValues(metricValues);
+ timelineMetrics.add(timelineMetric);
+ }
+ }
+
+ return timelineMetrics;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/RFunctionInvoker.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/RFunctionInvoker.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/RFunctionInvoker.java
new file mode 100644
index 0000000..8f1eba6
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/RFunctionInvoker.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.core;
+
+
+import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
+import org.apache.ambari.metrics.adservice.prototype.common.ResultSet;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.rosuda.JRI.REXP;
+import org.rosuda.JRI.RVector;
+import org.rosuda.JRI.Rengine;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class RFunctionInvoker {
+
+ static final Log LOG = LogFactory.getLog(RFunctionInvoker.class);
+ public static Rengine r = new Rengine(new String[]{"--no-save"}, false, null);
+ private static String rScriptDir = "/usr/lib/ambari-metrics-collector/R-scripts";
+
+ private static void loadDataSets(Rengine r, DataSeries trainData, DataSeries testData) {
+ r.assign("train_ts", trainData.ts);
+ r.assign("train_x", trainData.values);
+ r.eval("train_data <- data.frame(train_ts,train_x)");
+ r.eval("names(train_data) <- c(\"TS\", " + trainData.seriesName + ")");
+
+ r.assign("test_ts", testData.ts);
+ r.assign("test_x", testData.values);
+ r.eval("test_data <- data.frame(test_ts,test_x)");
+ r.eval("names(test_data) <- c(\"TS\", " + testData.seriesName + ")");
+ }
+
+ public static void setScriptsDir(String dir) {
+ rScriptDir = dir;
+ }
+
+ public static ResultSet executeMethod(String methodType, DataSeries trainData, DataSeries testData, Map<String, String> configs) {
+
+ ResultSet result;
+ switch (methodType) {
+ case "tukeys":
+ result = tukeys(trainData, testData, configs);
+ break;
+ case "ema":
+ result = ema_global(trainData, testData, configs);
+ break;
+ case "ks":
+ result = ksTest(trainData, testData, configs);
+ break;
+ case "hsdev":
+ result = hsdev(trainData, testData, configs);
+ break;
+ default:
+ result = tukeys(trainData, testData, configs);
+ break;
+ }
+ return result;
+ }
+
+ public static ResultSet tukeys(DataSeries trainData, DataSeries testData, Map<String, String> configs) {
+ try {
+
+ REXP exp1 = r.eval("source('" + rScriptDir + "/tukeys.r" + "')");
+
+ double n = Double.parseDouble(configs.get("tukeys.n"));
+ r.eval("n <- " + n);
+
+ loadDataSets(r, trainData, testData);
+
+ r.eval("an <- ams_tukeys(train_data, test_data, n)");
+ REXP exp = r.eval("an");
+ RVector cont = (RVector) exp.getContent();
+ List<double[]> result = new ArrayList();
+ for (int i = 0; i < cont.size(); i++) {
+ result.add(cont.at(i).asDoubleArray());
+ }
+ return new ResultSet(result);
+ } catch (Exception e) {
+ LOG.error(e);
+ } finally {
+ r.end();
+ }
+ return null;
+ }
+
+ public static ResultSet ema_global(DataSeries trainData, DataSeries testData, Map<String, String> configs) {
+ try {
+ r.eval("source('" + rScriptDir + "/ema.r" + "')");
+
+ int n = Integer.parseInt(configs.get("ema.n"));
+ r.eval("n <- " + n);
+
+ double w = Double.parseDouble(configs.get("ema.w"));
+ r.eval("w <- " + w);
+
+ loadDataSets(r, trainData, testData);
+
+ r.eval("an <- ema_global(train_data, test_data, w, n)");
+ REXP exp = r.eval("an");
+ RVector cont = (RVector) exp.getContent();
+ List<double[]> result = new ArrayList();
+ for (int i = 0; i < cont.size(); i++) {
+ result.add(cont.at(i).asDoubleArray());
+ }
+ return new ResultSet(result);
+
+ } catch (Exception e) {
+ LOG.error(e);
+ } finally {
+ r.end();
+ }
+ return null;
+ }
+
+ public static ResultSet ema_daily(DataSeries trainData, DataSeries testData, Map<String, String> configs) {
+ try {
+ r.eval("source('" + rScriptDir + "/ema.r" + "')");
+
+ int n = Integer.parseInt(configs.get("ema.n"));
+ r.eval("n <- " + n);
+
+ double w = Double.parseDouble(configs.get("ema.w"));
+ r.eval("w <- " + w);
+
+ loadDataSets(r, trainData, testData);
+
+ r.eval("an <- ema_daily(train_data, test_data, w, n)");
+ REXP exp = r.eval("an");
+ RVector cont = (RVector) exp.getContent();
+ List<double[]> result = new ArrayList();
+ for (int i = 0; i < cont.size(); i++) {
+ result.add(cont.at(i).asDoubleArray());
+ }
+ return new ResultSet(result);
+
+ } catch (Exception e) {
+ LOG.error(e);
+ } finally {
+ r.end();
+ }
+ return null;
+ }
+
+ public static ResultSet ksTest(DataSeries trainData, DataSeries testData, Map<String, String> configs) {
+ try {
+ r.eval("source('" + rScriptDir + "/kstest.r" + "')");
+
+ double p_value = Double.parseDouble(configs.get("ks.p_value"));
+ r.eval("p_value <- " + p_value);
+
+ loadDataSets(r, trainData, testData);
+
+ r.eval("an <- ams_ks(train_data, test_data, p_value)");
+ REXP exp = r.eval("an");
+ RVector cont = (RVector) exp.getContent();
+ List<double[]> result = new ArrayList();
+ for (int i = 0; i < cont.size(); i++) {
+ result.add(cont.at(i).asDoubleArray());
+ }
+ return new ResultSet(result);
+
+ } catch (Exception e) {
+ LOG.error(e);
+ } finally {
+ r.end();
+ }
+ return null;
+ }
+
+ public static ResultSet hsdev(DataSeries trainData, DataSeries testData, Map<String, String> configs) {
+ try {
+ r.eval("source('" + rScriptDir + "/hsdev.r" + "')");
+
+ int n = Integer.parseInt(configs.get("hsdev.n"));
+ r.eval("n <- " + n);
+
+ int nhp = Integer.parseInt(configs.get("hsdev.nhp"));
+ r.eval("nhp <- " + nhp);
+
+ long interval = Long.parseLong(configs.get("hsdev.interval"));
+ r.eval("interval <- " + interval);
+
+ long period = Long.parseLong(configs.get("hsdev.period"));
+ r.eval("period <- " + period);
+
+ loadDataSets(r, trainData, testData);
+
+ r.eval("an2 <- hsdev_daily(train_data, test_data, n, nhp, interval, period)");
+ REXP exp = r.eval("an2");
+ RVector cont = (RVector) exp.getContent();
+
+ List<double[]> result = new ArrayList();
+ for (int i = 0; i < cont.size(); i++) {
+ result.add(cont.at(i).asDoubleArray());
+ }
+ return new ResultSet(result);
+ } catch (Exception e) {
+ LOG.error(e);
+ } finally {
+ r.end();
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/TrendADSystem.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/TrendADSystem.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/TrendADSystem.java
new file mode 100644
index 0000000..f5ec83a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/TrendADSystem.java
@@ -0,0 +1,317 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.core;
+
+import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
+import org.apache.ambari.metrics.adservice.prototype.methods.hsdev.HsdevTechnique;
+import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
+import org.apache.ambari.metrics.adservice.prototype.methods.kstest.KSTechnique;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class TrendADSystem implements Serializable {
+
+ private MetricsCollectorInterface metricsCollectorInterface;
+ private List<TrendMetric> trendMetrics;
+
+ private long ksTestIntervalMillis = 10 * 60 * 1000;
+ private long ksTrainIntervalMillis = 10 * 60 * 1000;
+ private KSTechnique ksTechnique;
+
+ private HsdevTechnique hsdevTechnique;
+ private int hsdevNumHistoricalPeriods = 3;
+
+ private Map<KsSingleRunKey, MetricAnomaly> trackedKsAnomalies = new HashMap<>();
+ private static final Log LOG = LogFactory.getLog(TrendADSystem.class);
+ private String inputFile = "";
+
+ public TrendADSystem(MetricsCollectorInterface metricsCollectorInterface,
+ long ksTestIntervalMillis,
+ long ksTrainIntervalMillis,
+ int hsdevNumHistoricalPeriods) {
+
+ this.metricsCollectorInterface = metricsCollectorInterface;
+ this.ksTestIntervalMillis = ksTestIntervalMillis;
+ this.ksTrainIntervalMillis = ksTrainIntervalMillis;
+ this.hsdevNumHistoricalPeriods = hsdevNumHistoricalPeriods;
+
+ this.ksTechnique = new KSTechnique();
+ this.hsdevTechnique = new HsdevTechnique();
+
+ trendMetrics = new ArrayList<>();
+ }
+
+ public void runKSTest(long currentEndTime, Set<TrendMetric> trendMetrics) {
+ readInputFile(inputFile);
+
+ long ksTestIntervalStartTime = currentEndTime - ksTestIntervalMillis;
+ LOG.info("Running KS Test for test data interval [" + new Date(ksTestIntervalStartTime) + " : " +
+ new Date(currentEndTime) + "], with train data period [" + new Date(ksTestIntervalStartTime - ksTrainIntervalMillis)
+ + " : " + new Date(ksTestIntervalStartTime) + "]");
+
+ for (TrendMetric metric : trendMetrics) {
+ String metricName = metric.metricName;
+ String appId = metric.appId;
+ String hostname = metric.hostname;
+ String key = metricName + ":" + appId + ":" + hostname;
+
+ TimelineMetrics ksData = metricsCollectorInterface.fetchMetrics(metricName, appId, hostname, ksTestIntervalStartTime - ksTrainIntervalMillis,
+ currentEndTime);
+
+ if (ksData.getMetrics().isEmpty()) {
+ LOG.info("No metrics fetched for KS, metricKey = " + key);
+ continue;
+ }
+
+ List<Double> trainTsList = new ArrayList<>();
+ List<Double> trainDataList = new ArrayList<>();
+ List<Double> testTsList = new ArrayList<>();
+ List<Double> testDataList = new ArrayList<>();
+
+ for (TimelineMetric timelineMetric : ksData.getMetrics()) {
+ for (Long timestamp : timelineMetric.getMetricValues().keySet()) {
+ if (timestamp <= ksTestIntervalStartTime) {
+ trainDataList.add(timelineMetric.getMetricValues().get(timestamp));
+ trainTsList.add((double) timestamp);
+ } else {
+ testDataList.add(timelineMetric.getMetricValues().get(timestamp));
+ testTsList.add((double) timestamp);
+ }
+ }
+ }
+
+ LOG.info("Train Data size : " + trainDataList.size() + ", Test Data Size : " + testDataList.size());
+ if (trainDataList.isEmpty() || testDataList.isEmpty() || trainDataList.size() < testDataList.size()) {
+ LOG.info("Not enough train/test data to perform KS analysis.");
+ continue;
+ }
+
+ String ksTrainSeries = "KSTrainSeries";
+ double[] trainTs = new double[trainTsList.size()];
+ double[] trainData = new double[trainTsList.size()];
+ for (int i = 0; i < trainTs.length; i++) {
+ trainTs[i] = trainTsList.get(i);
+ trainData[i] = trainDataList.get(i);
+ }
+
+ String ksTestSeries = "KSTestSeries";
+ double[] testTs = new double[testTsList.size()];
+ double[] testData = new double[testTsList.size()];
+ for (int i = 0; i < testTs.length; i++) {
+ testTs[i] = testTsList.get(i);
+ testData[i] = testDataList.get(i);
+ }
+
+ LOG.info("Train Size = " + trainTs.length + ", Test Size = " + testTs.length);
+
+ DataSeries ksTrainData = new DataSeries(ksTrainSeries, trainTs, trainData);
+ DataSeries ksTestData = new DataSeries(ksTestSeries, testTs, testData);
+
+ MetricAnomaly metricAnomaly = ksTechnique.runKsTest(key, ksTrainData, ksTestData);
+ if (metricAnomaly == null) {
+ LOG.info("No anomaly from KS test.");
+ } else {
+ LOG.info("Found Anomaly in KS Test. Publishing KS Anomaly metric....");
+ TimelineMetric timelineMetric = getAsTimelineMetric(metricAnomaly,
+ ksTestIntervalStartTime, currentEndTime, ksTestIntervalStartTime - ksTrainIntervalMillis, ksTestIntervalStartTime);
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+ timelineMetrics.addOrMergeTimelineMetric(timelineMetric);
+ metricsCollectorInterface.emitMetrics(timelineMetrics);
+
+ trackedKsAnomalies.put(new KsSingleRunKey(ksTestIntervalStartTime, currentEndTime, metricName, appId, hostname), metricAnomaly);
+ }
+ }
+
+ if (trendMetrics.isEmpty()) {
+ LOG.info("No Trend metrics tracked!!!!");
+ }
+
+ }
+
+ private TimelineMetric getAsTimelineMetric(MetricAnomaly metricAnomaly,
+ long testStart,
+ long testEnd,
+ long trainStart,
+ long trainEnd) {
+
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setMetricName(metricAnomaly.getMetricKey());
+ timelineMetric.setAppId(MetricsCollectorInterface.serviceName + "-" + metricAnomaly.getMethodType());
+ timelineMetric.setInstanceId(null);
+ timelineMetric.setHostName(MetricsCollectorInterface.getDefaultLocalHostName());
+ timelineMetric.setStartTime(testEnd);
+ HashMap<String, String> metadata = new HashMap<>();
+ metadata.put("method", metricAnomaly.getMethodType());
+ metadata.put("anomaly-score", String.valueOf(metricAnomaly.getAnomalyScore()));
+ metadata.put("test-start-time", String.valueOf(testStart));
+ metadata.put("train-start-time", String.valueOf(trainStart));
+ metadata.put("train-end-time", String.valueOf(trainEnd));
+ timelineMetric.setMetadata(metadata);
+ TreeMap<Long,Double> metricValues = new TreeMap<>();
+ metricValues.put(testEnd, metricAnomaly.getMetricValue());
+ timelineMetric.setMetricValues(metricValues);
+ return timelineMetric;
+
+ }
+
+ public void runHsdevMethod() {
+
+ List<TimelineMetric> hsdevMetricAnomalies = new ArrayList<>();
+
+ for (KsSingleRunKey ksSingleRunKey : trackedKsAnomalies.keySet()) {
+
+ long hsdevTestEnd = ksSingleRunKey.endTime;
+ long hsdevTestStart = ksSingleRunKey.startTime;
+
+ long period = hsdevTestEnd - hsdevTestStart;
+
+ long hsdevTrainStart = hsdevTestStart - (hsdevNumHistoricalPeriods) * period;
+ long hsdevTrainEnd = hsdevTestStart;
+
+ LOG.info("Running HSdev Test for test data interval [" + new Date(hsdevTestStart) + " : " +
+ new Date(hsdevTestEnd) + "], with train data period [" + new Date(hsdevTrainStart)
+ + " : " + new Date(hsdevTrainEnd) + "]");
+
+ String metricName = ksSingleRunKey.metricName;
+ String appId = ksSingleRunKey.appId;
+ String hostname = ksSingleRunKey.hostname;
+ String key = metricName + "_" + appId + "_" + hostname;
+
+ TimelineMetrics hsdevData = metricsCollectorInterface.fetchMetrics(
+ metricName,
+ appId,
+ hostname,
+ hsdevTrainStart,
+ hsdevTestEnd);
+
+ if (hsdevData.getMetrics().isEmpty()) {
+ LOG.info("No metrics fetched for HSDev, metricKey = " + key);
+ continue;
+ }
+
+ List<Double> trainTsList = new ArrayList<>();
+ List<Double> trainDataList = new ArrayList<>();
+ List<Double> testTsList = new ArrayList<>();
+ List<Double> testDataList = new ArrayList<>();
+
+ for (TimelineMetric timelineMetric : hsdevData.getMetrics()) {
+ for (Long timestamp : timelineMetric.getMetricValues().keySet()) {
+ if (timestamp <= hsdevTestStart) {
+ trainDataList.add(timelineMetric.getMetricValues().get(timestamp));
+ trainTsList.add((double) timestamp);
+ } else {
+ testDataList.add(timelineMetric.getMetricValues().get(timestamp));
+ testTsList.add((double) timestamp);
+ }
+ }
+ }
+
+ if (trainDataList.isEmpty() || testDataList.isEmpty() || trainDataList.size() < testDataList.size()) {
+ LOG.info("Not enough train/test data to perform Hsdev analysis.");
+ continue;
+ }
+
+ String hsdevTrainSeries = "HsdevTrainSeries";
+ double[] trainTs = new double[trainTsList.size()];
+ double[] trainData = new double[trainTsList.size()];
+ for (int i = 0; i < trainTs.length; i++) {
+ trainTs[i] = trainTsList.get(i);
+ trainData[i] = trainDataList.get(i);
+ }
+
+ String hsdevTestSeries = "HsdevTestSeries";
+ double[] testTs = new double[testTsList.size()];
+ double[] testData = new double[testTsList.size()];
+ for (int i = 0; i < testTs.length; i++) {
+ testTs[i] = testTsList.get(i);
+ testData[i] = testDataList.get(i);
+ }
+
+ LOG.info("Train Size = " + trainTs.length + ", Test Size = " + testTs.length);
+
+ DataSeries hsdevTrainData = new DataSeries(hsdevTrainSeries, trainTs, trainData);
+ DataSeries hsdevTestData = new DataSeries(hsdevTestSeries, testTs, testData);
+
+ MetricAnomaly metricAnomaly = hsdevTechnique.runHsdevTest(key, hsdevTrainData, hsdevTestData);
+ if (metricAnomaly == null) {
+ LOG.info("No anomaly from Hsdev test. Mismatch between KS and HSDev. ");
+ ksTechnique.updateModel(key, false, 10);
+ } else {
+ LOG.info("Found Anomaly in Hsdev Test. This confirms KS anomaly.");
+ hsdevMetricAnomalies.add(getAsTimelineMetric(metricAnomaly,
+ hsdevTestStart, hsdevTestEnd, hsdevTrainStart, hsdevTrainEnd));
+ }
+ }
+ clearTrackedKsRunKeys();
+
+ if (!hsdevMetricAnomalies.isEmpty()) {
+ LOG.info("Publishing Hsdev Anomalies....");
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+ timelineMetrics.setMetrics(hsdevMetricAnomalies);
+ metricsCollectorInterface.emitMetrics(timelineMetrics);
+ }
+ }
+
+ private void clearTrackedKsRunKeys() {
+ trackedKsAnomalies.clear();
+ }
+
+ private void readInputFile(String fileName) {
+ trendMetrics.clear();
+ try (BufferedReader br = new BufferedReader(new FileReader(fileName))) {
+ for (String line; (line = br.readLine()) != null; ) {
+ String[] splits = line.split(",");
+ LOG.info("Adding a new metric to track in Trend AD system : " + splits[0]);
+ trendMetrics.add(new TrendMetric(splits[0], splits[1], splits[2]));
+ }
+ } catch (IOException e) {
+ LOG.error("Error reading input file : " + e);
+ }
+ }
+
+ class KsSingleRunKey implements Serializable{
+
+ long startTime;
+ long endTime;
+ String metricName;
+ String appId;
+ String hostname;
+
+ public KsSingleRunKey(long startTime, long endTime, String metricName, String appId, String hostname) {
+ this.startTime = startTime;
+ this.endTime = endTime;
+ this.metricName = metricName;
+ this.appId = appId;
+ this.hostname = hostname;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/TrendMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/TrendMetric.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/TrendMetric.java
new file mode 100644
index 0000000..d4db227
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/TrendMetric.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.core;
+
+import java.io.Serializable;
+
+public class TrendMetric implements Serializable {
+
+ String metricName;
+ String appId;
+ String hostname;
+
+ public TrendMetric(String metricName, String appId, String hostname) {
+ this.metricName = metricName;
+ this.appId = appId;
+ this.hostname = hostname;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/AnomalyDetectionTechnique.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/AnomalyDetectionTechnique.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/AnomalyDetectionTechnique.java
new file mode 100644
index 0000000..c19adda
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/AnomalyDetectionTechnique.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype.methods;
+
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+
+import java.util.List;
+
+public abstract class AnomalyDetectionTechnique {
+
+ protected String methodType;
+
+ public abstract List<MetricAnomaly> test(TimelineMetric metric);
+
+}
[4/6] ambari git commit: AMBARI-22077 : Create maven module and
package structure for the anomaly detection engine. (Commit 2) (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestTukeys.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestTukeys.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestTukeys.java
new file mode 100644
index 0000000..57a6f34
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/prototype/TestTukeys.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.prototype;
+
+import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
+import org.apache.ambari.metrics.adservice.prototype.core.MetricsCollectorInterface;
+import org.apache.ambari.metrics.adservice.prototype.core.RFunctionInvoker;
+import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaTechnique;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.List;
+import java.util.TreeMap;
+
+public class TestTukeys {
+
+ @BeforeClass
+ public static void init() throws URISyntaxException {
+ Assume.assumeTrue(System.getenv("R_HOME") != null);
+ }
+
+ @Test
+ public void testPointInTimeDetectionSystem() throws UnknownHostException, URISyntaxException {
+
+ URL url = ClassLoader.getSystemResource("R-scripts");
+ String fullFilePath = new File(url.toURI()).getAbsolutePath();
+ RFunctionInvoker.setScriptsDir(fullFilePath);
+
+ MetricsCollectorInterface metricsCollectorInterface = new MetricsCollectorInterface("avijayan-ams-1.openstacklocal","http", "6188");
+
+ EmaTechnique ema = new EmaTechnique(0.5, 3);
+ long now = System.currentTimeMillis();
+
+ TimelineMetric metric1 = new TimelineMetric();
+ metric1.setMetricName("mm9");
+ metric1.setHostName(MetricsCollectorInterface.getDefaultLocalHostName());
+ metric1.setStartTime(now);
+ metric1.setAppId("aa9");
+ metric1.setInstanceId(null);
+ metric1.setType("Integer");
+
+ //Train
+ TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+
+ //2hr data.
+ for (int i = 0; i < 120; i++) {
+ double metric = 20000 + Math.random();
+ metricValues.put(now - i * 60 * 1000, metric);
+ }
+ metric1.setMetricValues(metricValues);
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+ timelineMetrics.addOrMergeTimelineMetric(metric1);
+
+ metricsCollectorInterface.emitMetrics(timelineMetrics);
+
+ List<MetricAnomaly> anomalyList = ema.test(metric1);
+ metricsCollectorInterface.publish(anomalyList);
+//
+// PointInTimeADSystem pointInTimeADSystem = new PointInTimeADSystem(ema, metricsCollectorInterface, 3, 5*60*1000, 15*60*1000);
+// pointInTimeADSystem.runOnce();
+//
+// List<MetricAnomaly> anomalyList2 = ema.test(metric1);
+//
+// pointInTimeADSystem.runOnce();
+// List<MetricAnomaly> anomalyList3 = ema.test(metric1);
+//
+// pointInTimeADSystem.runOnce();
+// List<MetricAnomaly> anomalyList4 = ema.test(metric1);
+//
+// pointInTimeADSystem.runOnce();
+// List<MetricAnomaly> anomalyList5 = ema.test(metric1);
+//
+// pointInTimeADSystem.runOnce();
+// List<MetricAnomaly> anomalyList6 = ema.test(metric1);
+//
+// Assert.assertTrue(anomalyList6.size() < anomalyList.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/AbstractMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/AbstractMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/AbstractMetricSeries.java
new file mode 100644
index 0000000..635a929
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/AbstractMetricSeries.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.seriesgenerator;
+
+public interface AbstractMetricSeries {
+
+ public double nextValue();
+ public double[] getSeries(int n);
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/DualBandMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/DualBandMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/DualBandMetricSeries.java
new file mode 100644
index 0000000..a9e3f30
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/DualBandMetricSeries.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.seriesgenerator;
+
+import java.util.Random;
+
+public class DualBandMetricSeries implements AbstractMetricSeries {
+
+ double lowBandValue = 0.0;
+ double lowBandDeviationPercentage = 0.0;
+ int lowBandPeriodSize = 10;
+ double highBandValue = 1.0;
+ double highBandDeviationPercentage = 0.0;
+ int highBandPeriodSize = 10;
+
+ Random random = new Random();
+ double lowBandValueLowerLimit, lowBandValueHigherLimit;
+ double highBandLowerLimit, highBandUpperLimit;
+ int l = 0, h = 0;
+
+ public DualBandMetricSeries(double lowBandValue,
+ double lowBandDeviationPercentage,
+ int lowBandPeriodSize,
+ double highBandValue,
+ double highBandDeviationPercentage,
+ int highBandPeriodSize) {
+ this.lowBandValue = lowBandValue;
+ this.lowBandDeviationPercentage = lowBandDeviationPercentage;
+ this.lowBandPeriodSize = lowBandPeriodSize;
+ this.highBandValue = highBandValue;
+ this.highBandDeviationPercentage = highBandDeviationPercentage;
+ this.highBandPeriodSize = highBandPeriodSize;
+ init();
+ }
+
+ private void init() {
+ lowBandValueLowerLimit = lowBandValue - lowBandDeviationPercentage * lowBandValue;
+ lowBandValueHigherLimit = lowBandValue + lowBandDeviationPercentage * lowBandValue;
+ highBandLowerLimit = highBandValue - highBandDeviationPercentage * highBandValue;
+ highBandUpperLimit = highBandValue + highBandDeviationPercentage * highBandValue;
+ }
+
+ @Override
+ public double nextValue() {
+
+ double value = 0.0;
+
+ if (l < lowBandPeriodSize) {
+ value = lowBandValueLowerLimit + (lowBandValueHigherLimit - lowBandValueLowerLimit) * random.nextDouble();
+ l++;
+ } else if (h < highBandPeriodSize) {
+ value = highBandLowerLimit + (highBandUpperLimit - highBandLowerLimit) * random.nextDouble();
+ h++;
+ }
+
+ if (l == lowBandPeriodSize && h == highBandPeriodSize) {
+ l = 0;
+ h = 0;
+ }
+
+ return value;
+ }
+
+ @Override
+ public double[] getSeries(int n) {
+ double[] series = new double[n];
+ for (int i = 0; i < n; i++) {
+ series[i] = nextValue();
+ }
+ return series;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MetricSeriesGeneratorFactory.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MetricSeriesGeneratorFactory.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MetricSeriesGeneratorFactory.java
new file mode 100644
index 0000000..a50b433
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MetricSeriesGeneratorFactory.java
@@ -0,0 +1,377 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.adservice.seriesgenerator;
+
+import java.util.Map;
+
+public class MetricSeriesGeneratorFactory {
+
+ /**
+ * Return a normally distributed data series with some deviation % and outliers.
+ *
+ * @param n size of the data series
+ * @param value The value around which the uniform data series is centered on.
+ * @param deviationPercentage The allowed deviation % on either side of the uniform value. For example, if value = 10, and deviation % is 0.1, the series values lie between 0.9 to 1.1.
+ * @param outlierProbability The probability of finding an outlier in the series.
+ * @param outlierDeviationLowerPercentage min percentage outlier should be away from the uniform value in % terms. if value = 10 and outlierDeviationPercentage = 30%, the outlier is 7 and 13.
+ * @param outlierDeviationHigherPercentage max percentage outlier should be away from the uniform value in % terms. if value = 10 and outlierDeviationPercentage = 60%, the outlier is 4 and 16.
+ * @param outliersAboveValue Outlier should be greater or smaller than the value.
+ * @return uniform series
+ */
+ public static double[] createUniformSeries(int n,
+ double value,
+ double deviationPercentage,
+ double outlierProbability,
+ double outlierDeviationLowerPercentage,
+ double outlierDeviationHigherPercentage,
+ boolean outliersAboveValue) {
+
+ UniformMetricSeries metricSeries = new UniformMetricSeries(value,
+ deviationPercentage,
+ outlierProbability,
+ outlierDeviationLowerPercentage,
+ outlierDeviationHigherPercentage,
+ outliersAboveValue);
+
+ return metricSeries.getSeries(n);
+ }
+
+
+ /**
+ * /**
+ * Returns a normally distributed series.
+ *
+ * @param n size of the data series
+ * @param mean mean of the distribution
+ * @param sd sd of the distribution
+ * @param outlierProbability sd of the distribution
+ * @param outlierDeviationSDTimesLower Lower Limit of the outlier with respect to times sdev from the mean.
+ * @param outlierDeviationSDTimesHigher Higher Limit of the outlier with respect to times sdev from the mean.
+ * @param outlierOnRightEnd Outlier should be on the right end or the left end.
+ * @return normal series
+ */
+ public static double[] createNormalSeries(int n,
+ double mean,
+ double sd,
+ double outlierProbability,
+ double outlierDeviationSDTimesLower,
+ double outlierDeviationSDTimesHigher,
+ boolean outlierOnRightEnd) {
+
+
+ NormalMetricSeries metricSeries = new NormalMetricSeries(mean,
+ sd,
+ outlierProbability,
+ outlierDeviationSDTimesLower,
+ outlierDeviationSDTimesHigher,
+ outlierOnRightEnd);
+
+ return metricSeries.getSeries(n);
+ }
+
+
+ /**
+ * Returns a monotonically increasing / decreasing series
+ *
+ * @param n size of the data series
+ * @param startValue Start value of the monotonic sequence
+ * @param slope direction of monotonicity m > 0 for increasing and m < 0 for decreasing.
+ * @param deviationPercentage The allowed deviation % on either side of the current 'y' value. For example, if current value = 10 according to slope, and deviation % is 0.1, the series values lie between 0.9 to 1.1.
+ * @param outlierProbability The probability of finding an outlier in the series.
+ * @param outlierDeviationLowerPercentage min percentage outlier should be away from the current 'y' value in % terms. if value = 10 and outlierDeviationPercentage = 30%, the outlier is 7 and 13.
+ * @param outlierDeviationHigherPercentage max percentage outlier should be away from the current 'y' value in % terms. if value = 10 and outlierDeviationPercentage = 60%, the outlier is 4 and 16.
+ * @param outliersAboveValue Outlier should be greater or smaller than the 'y' value.
+ * @return
+ */
+ public static double[] createMonotonicSeries(int n,
+ double startValue,
+ double slope,
+ double deviationPercentage,
+ double outlierProbability,
+ double outlierDeviationLowerPercentage,
+ double outlierDeviationHigherPercentage,
+ boolean outliersAboveValue) {
+
+ MonotonicMetricSeries metricSeries = new MonotonicMetricSeries(startValue,
+ slope,
+ deviationPercentage,
+ outlierProbability,
+ outlierDeviationLowerPercentage,
+ outlierDeviationHigherPercentage,
+ outliersAboveValue);
+
+ return metricSeries.getSeries(n);
+ }
+
+
+ /**
+ * Returns a dual band series (lower and higher)
+ *
+ * @param n size of the data series
+ * @param lowBandValue lower band value
+ * @param lowBandDeviationPercentage lower band deviation
+ * @param lowBandPeriodSize lower band
+ * @param highBandValue high band centre value
+ * @param highBandDeviationPercentage high band deviation.
+ * @param highBandPeriodSize high band size
+ * @return
+ */
+ public static double[] getDualBandSeries(int n,
+ double lowBandValue,
+ double lowBandDeviationPercentage,
+ int lowBandPeriodSize,
+ double highBandValue,
+ double highBandDeviationPercentage,
+ int highBandPeriodSize) {
+
+ DualBandMetricSeries metricSeries = new DualBandMetricSeries(lowBandValue,
+ lowBandDeviationPercentage,
+ lowBandPeriodSize,
+ highBandValue,
+ highBandDeviationPercentage,
+ highBandPeriodSize);
+
+ return metricSeries.getSeries(n);
+ }
+
+ /**
+ * Returns a step function series.
+ *
+ * @param n size of the data series
+ * @param startValue start steady value
+ * @param steadyValueDeviationPercentage required devation in the steady state value
+ * @param steadyPeriodSlope direction of monotonicity m > 0 for increasing and m < 0 for decreasing, m = 0 no increase or decrease.
+ * @param steadyPeriodMinSize min size for step period
+ * @param steadyPeriodMaxSize max size for step period.
+ * @param stepChangePercentage Increase / decrease in steady state to denote a step in terms of deviation percentage from the last value.
+ * @param upwardStep upward or downward step.
+ * @return
+ */
+ public static double[] getStepFunctionSeries(int n,
+ double startValue,
+ double steadyValueDeviationPercentage,
+ double steadyPeriodSlope,
+ int steadyPeriodMinSize,
+ int steadyPeriodMaxSize,
+ double stepChangePercentage,
+ boolean upwardStep) {
+
+ StepFunctionMetricSeries metricSeries = new StepFunctionMetricSeries(startValue,
+ steadyValueDeviationPercentage,
+ steadyPeriodSlope,
+ steadyPeriodMinSize,
+ steadyPeriodMaxSize,
+ stepChangePercentage,
+ upwardStep);
+
+ return metricSeries.getSeries(n);
+ }
+
+ /**
+ * Series with small period of turbulence and then back to steady.
+ *
+ * @param n size of the data series
+ * @param steadyStateValue steady state center value
+ * @param steadyStateDeviationPercentage steady state deviation in percentage
+ * @param turbulentPeriodDeviationLowerPercentage turbulent state lower limit in terms of percentage from centre value.
+ * @param turbulentPeriodDeviationHigherPercentage turbulent state higher limit in terms of percentage from centre value.
+ * @param turbulentPeriodLength turbulent period length (number of points)
+ * @param turbulentStatePosition Where the turbulent state should be 0 - at the beginning, 1 - in the middle (25% - 50% of the series), 2 - at the end of the series.
+ * @return
+ */
+ public static double[] getSteadySeriesWithTurbulentPeriod(int n,
+ double steadyStateValue,
+ double steadyStateDeviationPercentage,
+ double turbulentPeriodDeviationLowerPercentage,
+ double turbulentPeriodDeviationHigherPercentage,
+ int turbulentPeriodLength,
+ int turbulentStatePosition
+ ) {
+
+
+ SteadyWithTurbulenceMetricSeries metricSeries = new SteadyWithTurbulenceMetricSeries(n,
+ steadyStateValue,
+ steadyStateDeviationPercentage,
+ turbulentPeriodDeviationLowerPercentage,
+ turbulentPeriodDeviationHigherPercentage,
+ turbulentPeriodLength,
+ turbulentStatePosition);
+
+ return metricSeries.getSeries(n);
+ }
+
+
+ public static double[] generateSeries(String type, int n, Map<String, String> configs) {
+
+ double[] series;
+ switch (type) {
+
+ case "normal":
+ series = createNormalSeries(n,
+ Double.parseDouble(configs.getOrDefault("mean", "0")),
+ Double.parseDouble(configs.getOrDefault("sd", "1")),
+ Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesLower", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesHigher", "0")),
+ Boolean.parseBoolean(configs.getOrDefault("outlierOnRightEnd", "true")));
+ break;
+
+ case "uniform":
+ series = createUniformSeries(n,
+ Double.parseDouble(configs.getOrDefault("value", "10")),
+ Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
+ Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
+ break;
+
+ case "monotonic":
+ series = createMonotonicSeries(n,
+ Double.parseDouble(configs.getOrDefault("startValue", "10")),
+ Double.parseDouble(configs.getOrDefault("slope", "0")),
+ Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
+ Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
+ break;
+
+ case "dualband":
+ series = getDualBandSeries(n,
+ Double.parseDouble(configs.getOrDefault("lowBandValue", "10")),
+ Double.parseDouble(configs.getOrDefault("lowBandDeviationPercentage", "0")),
+ Integer.parseInt(configs.getOrDefault("lowBandPeriodSize", "0")),
+ Double.parseDouble(configs.getOrDefault("highBandValue", "10")),
+ Double.parseDouble(configs.getOrDefault("highBandDeviationPercentage", "0")),
+ Integer.parseInt(configs.getOrDefault("highBandPeriodSize", "0")));
+ break;
+
+ case "step":
+ series = getStepFunctionSeries(n,
+ Double.parseDouble(configs.getOrDefault("startValue", "10")),
+ Double.parseDouble(configs.getOrDefault("steadyValueDeviationPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("steadyPeriodSlope", "0")),
+ Integer.parseInt(configs.getOrDefault("steadyPeriodMinSize", "0")),
+ Integer.parseInt(configs.getOrDefault("steadyPeriodMaxSize", "0")),
+ Double.parseDouble(configs.getOrDefault("stepChangePercentage", "0")),
+ Boolean.parseBoolean(configs.getOrDefault("upwardStep", "true")));
+ break;
+
+ case "turbulence":
+ series = getSteadySeriesWithTurbulentPeriod(n,
+ Double.parseDouble(configs.getOrDefault("steadyStateValue", "10")),
+ Double.parseDouble(configs.getOrDefault("steadyStateDeviationPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationLowerPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationHigherPercentage", "10")),
+ Integer.parseInt(configs.getOrDefault("turbulentPeriodLength", "0")),
+ Integer.parseInt(configs.getOrDefault("turbulentStatePosition", "0")));
+ break;
+
+ default:
+ series = createNormalSeries(n,
+ 0,
+ 1,
+ 0,
+ 0,
+ 0,
+ true);
+ }
+ return series;
+ }
+
+ public static AbstractMetricSeries generateSeries(String type, Map<String, String> configs) {
+
+ AbstractMetricSeries series;
+ switch (type) {
+
+ case "normal":
+ series = new NormalMetricSeries(Double.parseDouble(configs.getOrDefault("mean", "0")),
+ Double.parseDouble(configs.getOrDefault("sd", "1")),
+ Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesLower", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesHigher", "0")),
+ Boolean.parseBoolean(configs.getOrDefault("outlierOnRightEnd", "true")));
+ break;
+
+ case "uniform":
+ series = new UniformMetricSeries(
+ Double.parseDouble(configs.getOrDefault("value", "10")),
+ Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
+ Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
+ break;
+
+ case "monotonic":
+ series = new MonotonicMetricSeries(
+ Double.parseDouble(configs.getOrDefault("startValue", "10")),
+ Double.parseDouble(configs.getOrDefault("slope", "0")),
+ Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
+ Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
+ break;
+
+ case "dualband":
+ series = new DualBandMetricSeries(
+ Double.parseDouble(configs.getOrDefault("lowBandValue", "10")),
+ Double.parseDouble(configs.getOrDefault("lowBandDeviationPercentage", "0")),
+ Integer.parseInt(configs.getOrDefault("lowBandPeriodSize", "0")),
+ Double.parseDouble(configs.getOrDefault("highBandValue", "10")),
+ Double.parseDouble(configs.getOrDefault("highBandDeviationPercentage", "0")),
+ Integer.parseInt(configs.getOrDefault("highBandPeriodSize", "0")));
+ break;
+
+ case "step":
+ series = new StepFunctionMetricSeries(
+ Double.parseDouble(configs.getOrDefault("startValue", "10")),
+ Double.parseDouble(configs.getOrDefault("steadyValueDeviationPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("steadyPeriodSlope", "0")),
+ Integer.parseInt(configs.getOrDefault("steadyPeriodMinSize", "0")),
+ Integer.parseInt(configs.getOrDefault("steadyPeriodMaxSize", "0")),
+ Double.parseDouble(configs.getOrDefault("stepChangePercentage", "0")),
+ Boolean.parseBoolean(configs.getOrDefault("upwardStep", "true")));
+ break;
+
+ case "turbulence":
+ series = new SteadyWithTurbulenceMetricSeries(
+ Integer.parseInt(configs.getOrDefault("approxSeriesLength", "100")),
+ Double.parseDouble(configs.getOrDefault("steadyStateValue", "10")),
+ Double.parseDouble(configs.getOrDefault("steadyStateDeviationPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationLowerPercentage", "0")),
+ Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationHigherPercentage", "10")),
+ Integer.parseInt(configs.getOrDefault("turbulentPeriodLength", "0")),
+ Integer.parseInt(configs.getOrDefault("turbulentStatePosition", "0")));
+ break;
+
+ default:
+ series = new NormalMetricSeries(0,
+ 1,
+ 0,
+ 0,
+ 0,
+ true);
+ }
+ return series;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MetricSeriesGeneratorTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MetricSeriesGeneratorTest.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MetricSeriesGeneratorTest.java
new file mode 100644
index 0000000..03537e4
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MetricSeriesGeneratorTest.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.seriesgenerator;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class MetricSeriesGeneratorTest {
+
+ @Test
+ public void testUniformSeries() {
+
+ UniformMetricSeries metricSeries = new UniformMetricSeries(5, 0.2, 0, 0, 0, true);
+ Assert.assertTrue(metricSeries.nextValue() <= 6 && metricSeries.nextValue() >= 4);
+
+ double[] uniformSeries = MetricSeriesGeneratorFactory.createUniformSeries(50, 10, 0.2, 0.1, 0.4, 0.5, true);
+ Assert.assertTrue(uniformSeries.length == 50);
+
+ for (int i = 0; i < uniformSeries.length; i++) {
+ double value = uniformSeries[i];
+
+ if (value > 10 * 1.2) {
+ Assert.assertTrue(value >= 10 * 1.4 && value <= 10 * 1.6);
+ } else {
+ Assert.assertTrue(value >= 10 * 0.8 && value <= 10 * 1.2);
+ }
+ }
+ }
+
+ @Test
+ public void testNormalSeries() {
+ NormalMetricSeries metricSeries = new NormalMetricSeries(0, 1, 0, 0, 0, true);
+ Assert.assertTrue(metricSeries.nextValue() <= 3 && metricSeries.nextValue() >= -3);
+ }
+
+ @Test
+ public void testMonotonicSeries() {
+
+ MonotonicMetricSeries metricSeries = new MonotonicMetricSeries(0, 0.5, 0, 0, 0, 0, true);
+ Assert.assertTrue(metricSeries.nextValue() == 0);
+ Assert.assertTrue(metricSeries.nextValue() == 0.5);
+
+ double[] incSeries = MetricSeriesGeneratorFactory.createMonotonicSeries(20, 0, 0.5, 0, 0, 0, 0, true);
+ Assert.assertTrue(incSeries.length == 20);
+ for (int i = 0; i < incSeries.length; i++) {
+ Assert.assertTrue(incSeries[i] == i * 0.5);
+ }
+ }
+
+ @Test
+ public void testDualBandSeries() {
+ double[] dualBandSeries = MetricSeriesGeneratorFactory.getDualBandSeries(30, 5, 0.2, 5, 15, 0.3, 4);
+ Assert.assertTrue(dualBandSeries[0] >= 4 && dualBandSeries[0] <= 6);
+ Assert.assertTrue(dualBandSeries[4] >= 4 && dualBandSeries[4] <= 6);
+ Assert.assertTrue(dualBandSeries[5] >= 10.5 && dualBandSeries[5] <= 19.5);
+ Assert.assertTrue(dualBandSeries[8] >= 10.5 && dualBandSeries[8] <= 19.5);
+ Assert.assertTrue(dualBandSeries[9] >= 4 && dualBandSeries[9] <= 6);
+ }
+
+ @Test
+ public void testStepSeries() {
+ double[] stepSeries = MetricSeriesGeneratorFactory.getStepFunctionSeries(30, 10, 0, 0, 5, 5, 0.5, true);
+
+ Assert.assertTrue(stepSeries[0] == 10);
+ Assert.assertTrue(stepSeries[4] == 10);
+
+ Assert.assertTrue(stepSeries[5] == 10*1.5);
+ Assert.assertTrue(stepSeries[9] == 10*1.5);
+
+ Assert.assertTrue(stepSeries[10] == 10*1.5*1.5);
+ Assert.assertTrue(stepSeries[14] == 10*1.5*1.5);
+ }
+
+ @Test
+ public void testSteadySeriesWithTurbulence() {
+ double[] steadySeriesWithTurbulence = MetricSeriesGeneratorFactory.getSteadySeriesWithTurbulentPeriod(30, 5, 0, 1, 1, 5, 1);
+
+ int count = 0;
+ for (int i = 0; i < steadySeriesWithTurbulence.length; i++) {
+ if (steadySeriesWithTurbulence[i] == 10) {
+ count++;
+ }
+ }
+ Assert.assertTrue(count == 5);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MonotonicMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MonotonicMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MonotonicMetricSeries.java
new file mode 100644
index 0000000..8bd1a9b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/MonotonicMetricSeries.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.seriesgenerator;
+
+import java.util.Random;
+
+public class MonotonicMetricSeries implements AbstractMetricSeries {
+
+ double startValue = 0.0;
+ double slope = 0.5;
+ double deviationPercentage = 0.0;
+ double outlierProbability = 0.0;
+ double outlierDeviationLowerPercentage = 0.0;
+ double outlierDeviationHigherPercentage = 0.0;
+ boolean outliersAboveValue = true;
+
+ Random random = new Random();
+ double nonOutlierProbability;
+
+ // y = mx + c
+ double y;
+ double m;
+ double x;
+ double c;
+
+ public MonotonicMetricSeries(double startValue,
+ double slope,
+ double deviationPercentage,
+ double outlierProbability,
+ double outlierDeviationLowerPercentage,
+ double outlierDeviationHigherPercentage,
+ boolean outliersAboveValue) {
+ this.startValue = startValue;
+ this.slope = slope;
+ this.deviationPercentage = deviationPercentage;
+ this.outlierProbability = outlierProbability;
+ this.outlierDeviationLowerPercentage = outlierDeviationLowerPercentage;
+ this.outlierDeviationHigherPercentage = outlierDeviationHigherPercentage;
+ this.outliersAboveValue = outliersAboveValue;
+ init();
+ }
+
+ private void init() {
+ y = startValue;
+ m = slope;
+ x = 1;
+ c = y - (m * x);
+ nonOutlierProbability = 1.0 - outlierProbability;
+ }
+
+ @Override
+ public double nextValue() {
+
+ double value;
+ double probability = random.nextDouble();
+
+ y = m * x + c;
+ if (probability <= nonOutlierProbability) {
+ double valueDeviationLowerLimit = y - deviationPercentage * y;
+ double valueDeviationHigherLimit = y + deviationPercentage * y;
+ value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
+ } else {
+ if (outliersAboveValue) {
+ double outlierLowerLimit = y + outlierDeviationLowerPercentage * y;
+ double outlierUpperLimit = y + outlierDeviationHigherPercentage * y;
+ value = outlierLowerLimit + (outlierUpperLimit - outlierLowerLimit) * random.nextDouble();
+ } else {
+ double outlierLowerLimit = y - outlierDeviationLowerPercentage * y;
+ double outlierUpperLimit = y - outlierDeviationHigherPercentage * y;
+ value = outlierUpperLimit + (outlierLowerLimit - outlierUpperLimit) * random.nextDouble();
+ }
+ }
+ x++;
+ return value;
+ }
+
+ @Override
+ public double[] getSeries(int n) {
+ double[] series = new double[n];
+ for (int i = 0; i < n; i++) {
+ series[i] = nextValue();
+ }
+ return series;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/NormalMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/NormalMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/NormalMetricSeries.java
new file mode 100644
index 0000000..fdedb6e
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/NormalMetricSeries.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.seriesgenerator;
+
+import java.util.Random;
+
+public class NormalMetricSeries implements AbstractMetricSeries {
+
+ double mean = 0.0;
+ double sd = 1.0;
+ double outlierProbability = 0.0;
+ double outlierDeviationSDTimesLower = 0.0;
+ double outlierDeviationSDTimesHigher = 0.0;
+ boolean outlierOnRightEnd = true;
+
+ Random random = new Random();
+ double nonOutlierProbability;
+
+
+ public NormalMetricSeries(double mean,
+ double sd,
+ double outlierProbability,
+ double outlierDeviationSDTimesLower,
+ double outlierDeviationSDTimesHigher,
+ boolean outlierOnRightEnd) {
+ this.mean = mean;
+ this.sd = sd;
+ this.outlierProbability = outlierProbability;
+ this.outlierDeviationSDTimesLower = outlierDeviationSDTimesLower;
+ this.outlierDeviationSDTimesHigher = outlierDeviationSDTimesHigher;
+ this.outlierOnRightEnd = outlierOnRightEnd;
+ init();
+ }
+
+ private void init() {
+ nonOutlierProbability = 1.0 - outlierProbability;
+ }
+
+ @Override
+ public double nextValue() {
+
+ double value;
+ double probability = random.nextDouble();
+
+ if (probability <= nonOutlierProbability) {
+ value = random.nextGaussian() * sd + mean;
+ } else {
+ if (outlierOnRightEnd) {
+ value = mean + (outlierDeviationSDTimesLower + (outlierDeviationSDTimesHigher - outlierDeviationSDTimesLower) * random.nextDouble()) * sd;
+ } else {
+ value = mean - (outlierDeviationSDTimesLower + (outlierDeviationSDTimesHigher - outlierDeviationSDTimesLower) * random.nextDouble()) * sd;
+ }
+ }
+ return value;
+ }
+
+ @Override
+ public double[] getSeries(int n) {
+ double[] series = new double[n];
+ for (int i = 0; i < n; i++) {
+ series[i] = nextValue();
+ }
+ return series;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java
new file mode 100644
index 0000000..403e599
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.seriesgenerator;
+
+import java.util.Random;
+
+public class SteadyWithTurbulenceMetricSeries implements AbstractMetricSeries {
+
+ double steadyStateValue = 0.0;
+ double steadyStateDeviationPercentage = 0.0;
+ double turbulentPeriodDeviationLowerPercentage = 0.3;
+ double turbulentPeriodDeviationHigherPercentage = 0.5;
+ int turbulentPeriodLength = 5;
+ int turbulentStatePosition = 1;
+ int approximateSeriesLength = 10;
+
+ Random random = new Random();
+ double valueDeviationLowerLimit;
+ double valueDeviationHigherLimit;
+ double tPeriodLowerLimit;
+ double tPeriodUpperLimit;
+ int tPeriodStartIndex = 0;
+ int index = 0;
+
+ public SteadyWithTurbulenceMetricSeries(int approximateSeriesLength,
+ double steadyStateValue,
+ double steadyStateDeviationPercentage,
+ double turbulentPeriodDeviationLowerPercentage,
+ double turbulentPeriodDeviationHigherPercentage,
+ int turbulentPeriodLength,
+ int turbulentStatePosition) {
+ this.approximateSeriesLength = approximateSeriesLength;
+ this.steadyStateValue = steadyStateValue;
+ this.steadyStateDeviationPercentage = steadyStateDeviationPercentage;
+ this.turbulentPeriodDeviationLowerPercentage = turbulentPeriodDeviationLowerPercentage;
+ this.turbulentPeriodDeviationHigherPercentage = turbulentPeriodDeviationHigherPercentage;
+ this.turbulentPeriodLength = turbulentPeriodLength;
+ this.turbulentStatePosition = turbulentStatePosition;
+ init();
+ }
+
+ private void init() {
+
+ if (turbulentStatePosition == 1) {
+ tPeriodStartIndex = (int) (0.25 * approximateSeriesLength + (0.25 * approximateSeriesLength * random.nextDouble()));
+ } else if (turbulentStatePosition == 2) {
+ tPeriodStartIndex = approximateSeriesLength - turbulentPeriodLength;
+ }
+
+ valueDeviationLowerLimit = steadyStateValue - steadyStateDeviationPercentage * steadyStateValue;
+ valueDeviationHigherLimit = steadyStateValue + steadyStateDeviationPercentage * steadyStateValue;
+
+ tPeriodLowerLimit = steadyStateValue + turbulentPeriodDeviationLowerPercentage * steadyStateValue;
+ tPeriodUpperLimit = steadyStateValue + turbulentPeriodDeviationHigherPercentage * steadyStateValue;
+ }
+
+ @Override
+ public double nextValue() {
+
+ double value;
+
+ if (index >= tPeriodStartIndex && index <= (tPeriodStartIndex + turbulentPeriodLength)) {
+ value = tPeriodLowerLimit + (tPeriodUpperLimit - tPeriodLowerLimit) * random.nextDouble();
+ } else {
+ value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
+ }
+ index++;
+ return value;
+ }
+
+ @Override
+ public double[] getSeries(int n) {
+
+ double[] series = new double[n];
+ int turbulentPeriodStartIndex = 0;
+
+ if (turbulentStatePosition == 1) {
+ turbulentPeriodStartIndex = (int) (0.25 * n + (0.25 * n * random.nextDouble()));
+ } else if (turbulentStatePosition == 2) {
+ turbulentPeriodStartIndex = n - turbulentPeriodLength;
+ }
+
+ double valueDevLowerLimit = steadyStateValue - steadyStateDeviationPercentage * steadyStateValue;
+ double valueDevHigherLimit = steadyStateValue + steadyStateDeviationPercentage * steadyStateValue;
+
+ double turbulentPeriodLowerLimit = steadyStateValue + turbulentPeriodDeviationLowerPercentage * steadyStateValue;
+ double turbulentPeriodUpperLimit = steadyStateValue + turbulentPeriodDeviationHigherPercentage * steadyStateValue;
+
+ for (int i = 0; i < n; i++) {
+ if (i >= turbulentPeriodStartIndex && i < (turbulentPeriodStartIndex + turbulentPeriodLength)) {
+ series[i] = turbulentPeriodLowerLimit + (turbulentPeriodUpperLimit - turbulentPeriodLowerLimit) * random.nextDouble();
+ } else {
+ series[i] = valueDevLowerLimit + (valueDevHigherLimit - valueDevLowerLimit) * random.nextDouble();
+ }
+ }
+
+ return series;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/StepFunctionMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/StepFunctionMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/StepFunctionMetricSeries.java
new file mode 100644
index 0000000..c91eac9
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/StepFunctionMetricSeries.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.seriesgenerator;
+
+import java.util.Random;
+
+public class StepFunctionMetricSeries implements AbstractMetricSeries {
+
+ double startValue = 0.0;
+ double steadyValueDeviationPercentage = 0.0;
+ double steadyPeriodSlope = 0.5;
+ int steadyPeriodMinSize = 10;
+ int steadyPeriodMaxSize = 20;
+ double stepChangePercentage = 0.0;
+ boolean upwardStep = true;
+
+ Random random = new Random();
+
+ // y = mx + c
+ double y;
+ double m;
+ double x;
+ double c;
+ int currentStepSize;
+ int currentIndex;
+
+ public StepFunctionMetricSeries(double startValue,
+ double steadyValueDeviationPercentage,
+ double steadyPeriodSlope,
+ int steadyPeriodMinSize,
+ int steadyPeriodMaxSize,
+ double stepChangePercentage,
+ boolean upwardStep) {
+ this.startValue = startValue;
+ this.steadyValueDeviationPercentage = steadyValueDeviationPercentage;
+ this.steadyPeriodSlope = steadyPeriodSlope;
+ this.steadyPeriodMinSize = steadyPeriodMinSize;
+ this.steadyPeriodMaxSize = steadyPeriodMaxSize;
+ this.stepChangePercentage = stepChangePercentage;
+ this.upwardStep = upwardStep;
+ init();
+ }
+
+ private void init() {
+ y = startValue;
+ m = steadyPeriodSlope;
+ x = 1;
+ c = y - (m * x);
+
+ currentStepSize = (int) (steadyPeriodMinSize + (steadyPeriodMaxSize - steadyPeriodMinSize) * random.nextDouble());
+ currentIndex = 0;
+ }
+
+ @Override
+ public double nextValue() {
+
+ double value = 0.0;
+
+ if (currentIndex < currentStepSize) {
+ y = m * x + c;
+ double valueDeviationLowerLimit = y - steadyValueDeviationPercentage * y;
+ double valueDeviationHigherLimit = y + steadyValueDeviationPercentage * y;
+ value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
+ x++;
+ currentIndex++;
+ }
+
+ if (currentIndex == currentStepSize) {
+ currentIndex = 0;
+ currentStepSize = (int) (steadyPeriodMinSize + (steadyPeriodMaxSize - steadyPeriodMinSize) * random.nextDouble());
+ if (upwardStep) {
+ y = y + stepChangePercentage * y;
+ } else {
+ y = y - stepChangePercentage * y;
+ }
+ x = 1;
+ c = y - (m * x);
+ }
+
+ return value;
+ }
+
+ @Override
+ public double[] getSeries(int n) {
+ double[] series = new double[n];
+ for (int i = 0; i < n; i++) {
+ series[i] = nextValue();
+ }
+ return series;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/UniformMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/UniformMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/UniformMetricSeries.java
new file mode 100644
index 0000000..6122f82
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/java/org/apache/ambari/metrics/adservice/seriesgenerator/UniformMetricSeries.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.adservice.seriesgenerator;
+
+import java.util.Random;
+
+public class UniformMetricSeries implements AbstractMetricSeries {
+
+ double value = 0.0;
+ double deviationPercentage = 0.0;
+ double outlierProbability = 0.0;
+ double outlierDeviationLowerPercentage = 0.0;
+ double outlierDeviationHigherPercentage = 0.0;
+ boolean outliersAboveValue= true;
+
+ Random random = new Random();
+ double valueDeviationLowerLimit;
+ double valueDeviationHigherLimit;
+ double outlierLeftLowerLimit;
+ double outlierLeftHigherLimit;
+ double outlierRightLowerLimit;
+ double outlierRightUpperLimit;
+ double nonOutlierProbability;
+
+
+ public UniformMetricSeries(double value,
+ double deviationPercentage,
+ double outlierProbability,
+ double outlierDeviationLowerPercentage,
+ double outlierDeviationHigherPercentage,
+ boolean outliersAboveValue) {
+ this.value = value;
+ this.deviationPercentage = deviationPercentage;
+ this.outlierProbability = outlierProbability;
+ this.outlierDeviationLowerPercentage = outlierDeviationLowerPercentage;
+ this.outlierDeviationHigherPercentage = outlierDeviationHigherPercentage;
+ this.outliersAboveValue = outliersAboveValue;
+ init();
+ }
+
+ private void init() {
+ valueDeviationLowerLimit = value - deviationPercentage * value;
+ valueDeviationHigherLimit = value + deviationPercentage * value;
+
+ outlierLeftLowerLimit = value - outlierDeviationHigherPercentage * value;
+ outlierLeftHigherLimit = value - outlierDeviationLowerPercentage * value;
+ outlierRightLowerLimit = value + outlierDeviationLowerPercentage * value;
+ outlierRightUpperLimit = value + outlierDeviationHigherPercentage * value;
+
+ nonOutlierProbability = 1.0 - outlierProbability;
+ }
+
+ @Override
+ public double nextValue() {
+
+ double value;
+ double probability = random.nextDouble();
+
+ if (probability <= nonOutlierProbability) {
+ value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
+ } else {
+ if (!outliersAboveValue) {
+ value = outlierLeftLowerLimit + (outlierLeftHigherLimit - outlierLeftLowerLimit) * random.nextDouble();
+ } else {
+ value = outlierRightLowerLimit + (outlierRightUpperLimit - outlierRightLowerLimit) * random.nextDouble();
+ }
+ }
+ return value;
+ }
+
+ @Override
+ public double[] getSeries(int n) {
+ double[] series = new double[n];
+ for (int i = 0; i < n; i++) {
+ series[i] = nextValue();
+ }
+ return series;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/pom.xml b/ambari-metrics/ambari-metrics-anomaly-detector/pom.xml
deleted file mode 100644
index e6e12f2..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/pom.xml
+++ /dev/null
@@ -1,205 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>ambari-metrics</artifactId>
- <groupId>org.apache.ambari</groupId>
- <version>2.0.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>ambari-metrics-anomaly-detector</artifactId>
- <version>2.0.0.0-SNAPSHOT</version>
- <properties>
- <scala.version>2.10.4</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
- </properties>
-
- <repositories>
- <repository>
- <id>scala-tools.org</id>
- <name>Scala-Tools Maven2 Repository</name>
- <url>http://scala-tools.org/repo-releases</url>
- </repository>
- </repositories>
-
- <pluginRepositories>
- <pluginRepository>
- <id>scala-tools.org</id>
- <name>Scala-Tools Maven2 Repository</name>
- <url>http://scala-tools.org/repo-releases</url>
- </pluginRepository>
- </pluginRepositories>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.scala-tools</groupId>
- <artifactId>maven-scala-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <scalaVersion>${scala.version}</scalaVersion>
- <args>
- <arg>-target:jvm-1.5</arg>
- </args>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <name>Ambari Metrics Anomaly Detector</name>
- <packaging>jar</packaging>
-
- <dependencies>
-
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.5</version>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>1.7.2</version>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>1.7.2</version>
- </dependency>
-
- <dependency>
- <groupId>com.github.lucarosellini.rJava</groupId>
- <artifactId>JRI</artifactId>
- <version>0.9-7</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_2.11</artifactId>
- <version>2.1.1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>0.10.1.0</version>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jdmk</groupId>
- <artifactId>jmxtools</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jmx</groupId>
- <artifactId>jmxri</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.mail</groupId>
- <artifactId>mail</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jms</groupId>
- <artifactId>jmx</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jms</groupId>
- <artifactId>jms</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>0.10.1.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>connect-json</artifactId>
- <version>0.10.1.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-kafka_2.10</artifactId>
- <version>1.6.3</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
- <version>1.6.3</version>
- </dependency>
- <dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-spark</artifactId>
- <version>4.10.0-HBase-1.1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-mllib_2.10</artifactId>
- <version>1.3.0</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- <version>4.10</version>
- </dependency>
- <dependency>
- <groupId>org.apache.ambari</groupId>
- <artifactId>ambari-metrics-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.2.5</version>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>2.1.1</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-mllib_${scala.binary.version}</artifactId>
- <version>2.1.1</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/DataSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/DataSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/DataSeries.java
deleted file mode 100644
index eb19857..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/DataSeries.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.common;
-
-import java.util.Arrays;
-
-public class DataSeries {
-
- public String seriesName;
- public double[] ts;
- public double[] values;
-
- public DataSeries(String seriesName, double[] ts, double[] values) {
- this.seriesName = seriesName;
- this.ts = ts;
- this.values = values;
- }
-
- @Override
- public String toString() {
- return seriesName + Arrays.toString(ts) + Arrays.toString(values);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/ResultSet.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/ResultSet.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/ResultSet.java
deleted file mode 100644
index 101b0e9..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/ResultSet.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.common;
-
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ResultSet {
-
- public List<double[]> resultset = new ArrayList<>();
-
- public ResultSet(List<double[]> resultset) {
- this.resultset = resultset;
- }
-
- public void print() {
- System.out.println("Result : ");
- if (!resultset.isEmpty()) {
- for (int i = 0; i<resultset.get(0).length;i++) {
- for (double[] entity : resultset) {
- System.out.print(entity[i] + " ");
- }
- System.out.println();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/StatisticUtils.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/StatisticUtils.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/StatisticUtils.java
deleted file mode 100644
index 4ea4ac5..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/common/StatisticUtils.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.common;
-
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-
-public class StatisticUtils {
-
- public static double mean(double[] values) {
- double sum = 0;
- for (double d : values) {
- sum += d;
- }
- return sum / values.length;
- }
-
- public static double variance(double[] values) {
- double avg = mean(values);
- double variance = 0;
- for (double d : values) {
- variance += Math.pow(d - avg, 2.0);
- }
- return variance;
- }
-
- public static double sdev(double[] values, boolean useBesselsCorrection) {
- double variance = variance(values);
- int n = (useBesselsCorrection) ? values.length - 1 : values.length;
- return Math.sqrt(variance / n);
- }
-
- public static double median(double[] values) {
- double[] clonedValues = Arrays.copyOf(values, values.length);
- Arrays.sort(clonedValues);
- int n = values.length;
-
- if (n % 2 != 0) {
- return clonedValues[(n-1)/2];
- } else {
- return ( clonedValues[(n-1)/2] + clonedValues[n/2] ) / 2;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/AmbariServerInterface.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/AmbariServerInterface.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/AmbariServerInterface.java
deleted file mode 100644
index b6b1bf5..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/AmbariServerInterface.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.alertservice.prototype.core;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Serializable;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-
-public class AmbariServerInterface implements Serializable{
-
- private static final Log LOG = LogFactory.getLog(AmbariServerInterface.class);
-
- private String ambariServerHost;
- private String clusterName;
-
- public AmbariServerInterface(String ambariServerHost, String clusterName) {
- this.ambariServerHost = ambariServerHost;
- this.clusterName = clusterName;
- }
-
- public int getPointInTimeSensitivity() {
-
- String url = constructUri("http", ambariServerHost, "8080", "/api/v1/clusters/" + clusterName + "/alert_definitions?fields=*");
-
- URL obj = null;
- BufferedReader in = null;
-
- try {
- obj = new URL(url);
- HttpURLConnection con = (HttpURLConnection) obj.openConnection();
- con.setRequestMethod("GET");
-
- String encoded = Base64.getEncoder().encodeToString(("admin:admin").getBytes(StandardCharsets.UTF_8));
- con.setRequestProperty("Authorization", "Basic "+encoded);
-
- int responseCode = con.getResponseCode();
- LOG.info("Sending 'GET' request to URL : " + url);
- LOG.info("Response Code : " + responseCode);
-
- in = new BufferedReader(
- new InputStreamReader(con.getInputStream()));
-
- StringBuilder responseJsonSb = new StringBuilder();
- String line;
- while ((line = in.readLine()) != null) {
- responseJsonSb.append(line);
- }
-
- JSONObject jsonObject = new JSONObject(responseJsonSb.toString());
- JSONArray array = jsonObject.getJSONArray("items");
- for(int i = 0 ; i < array.length() ; i++){
- JSONObject alertDefn = array.getJSONObject(i).getJSONObject("AlertDefinition");
- if (alertDefn.get("name") != null && alertDefn.get("name").equals("point_in_time_metrics_anomalies")) {
- JSONObject sourceNode = alertDefn.getJSONObject("source");
- JSONArray params = sourceNode.getJSONArray("parameters");
- for(int j = 0 ; j < params.length() ; j++){
- JSONObject param = params.getJSONObject(j);
- if (param.get("name").equals("sensitivity")) {
- return param.getInt("value");
- }
- }
- break;
- }
- }
-
- } catch (Exception e) {
- LOG.error(e);
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- LOG.warn(e);
- }
- }
- }
-
- return -1;
- }
-
- private String constructUri(String protocol, String host, String port, String path) {
- StringBuilder sb = new StringBuilder(protocol);
- sb.append("://");
- sb.append(host);
- sb.append(":");
- sb.append(port);
- sb.append(path);
- return sb.toString();
- }
-
-// public static void main(String[] args) {
-// AmbariServerInterface ambariServerInterface = new AmbariServerInterface();
-// ambariServerInterface.getPointInTimeSensitivity("avijayan-ams-1.openstacklocal","c1");
-// }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricKafkaProducer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricKafkaProducer.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricKafkaProducer.java
deleted file mode 100644
index 2287ee3..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/core/MetricKafkaProducer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.core;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.clients.producer.RecordMetadata;
-
-import java.util.Properties;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-
-public class MetricKafkaProducer {
-
- Producer producer;
- private static String topicName = "ambari-metrics-topic";
-
- public MetricKafkaProducer(String kafkaServers) {
- Properties configProperties = new Properties();
- configProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServers); //"avijayan-ams-2.openstacklocal:6667"
- configProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.ByteArraySerializer");
- configProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.connect.json.JsonSerializer");
- producer = new KafkaProducer(configProperties);
- }
-
- public void sendMetrics(TimelineMetrics timelineMetrics) throws InterruptedException, ExecutionException {
-
- ObjectMapper objectMapper = new ObjectMapper();
- JsonNode jsonNode = objectMapper.valueToTree(timelineMetrics);
- ProducerRecord<String, JsonNode> rec = new ProducerRecord<String, JsonNode>(topicName,jsonNode);
- Future<RecordMetadata> kafkaFuture = producer.send(rec);
-
- System.out.println(kafkaFuture.isDone());
- System.out.println(kafkaFuture.get().topic());
- }
-}
[2/6] ambari git commit: AMBARI-22077 : Create maven module and
package structure for the anomaly detection engine. (Commit 2) (avijayan)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java
deleted file mode 100644
index 6485ebb..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.alertservice.prototype.testing.utilities;
-
-import org.apache.ambari.metrics.alertservice.prototype.core.MetricsCollectorInterface;
-import org.apache.ambari.metrics.alertservice.prototype.core.RFunctionInvoker;
-import org.apache.ambari.metrics.alertservice.prototype.common.DataSeries;
-import org.apache.ambari.metrics.alertservice.prototype.common.ResultSet;
-import org.apache.ambari.metrics.alertservice.seriesgenerator.MetricSeriesGeneratorFactory;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TreeMap;
-
-public class MetricAnomalyTester {
-
- public static String appId = MetricsCollectorInterface.serviceName;
- static final Log LOG = LogFactory.getLog(MetricAnomalyTester.class);
- static Map<String, TimelineMetric> timelineMetricMap = new HashMap<>();
-
- public static TimelineMetrics runTestAnomalyRequest(MetricAnomalyDetectorTestInput input) throws UnknownHostException {
-
- long currentTime = System.currentTimeMillis();
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- String hostname = InetAddress.getLocalHost().getHostName();
-
- //Train data
- TimelineMetric metric1 = new TimelineMetric();
- if (StringUtils.isNotEmpty(input.getTrainDataName())) {
- metric1 = timelineMetricMap.get(input.getTrainDataName());
- if (metric1 == null) {
- metric1 = new TimelineMetric();
- double[] trainSeries = MetricSeriesGeneratorFactory.generateSeries(input.getTrainDataType(), input.getTrainDataSize(), input.getTrainDataConfigs());
- metric1.setMetricName(input.getTrainDataName());
- metric1.setAppId(appId);
- metric1.setHostName(hostname);
- metric1.setStartTime(currentTime);
- metric1.setInstanceId(null);
- metric1.setMetricValues(getAsTimeSeries(currentTime, trainSeries));
- timelineMetricMap.put(input.getTrainDataName(), metric1);
- }
- timelineMetrics.getMetrics().add(metric1);
- } else {
- LOG.error("No train data name specified");
- }
-
- //Test data
- TimelineMetric metric2 = new TimelineMetric();
- if (StringUtils.isNotEmpty(input.getTestDataName())) {
- metric2 = timelineMetricMap.get(input.getTestDataName());
- if (metric2 == null) {
- metric2 = new TimelineMetric();
- double[] testSeries = MetricSeriesGeneratorFactory.generateSeries(input.getTestDataType(), input.getTestDataSize(), input.getTestDataConfigs());
- metric2.setMetricName(input.getTestDataName());
- metric2.setAppId(appId);
- metric2.setHostName(hostname);
- metric2.setStartTime(currentTime);
- metric2.setInstanceId(null);
- metric2.setMetricValues(getAsTimeSeries(currentTime, testSeries));
- timelineMetricMap.put(input.getTestDataName(), metric2);
- }
- timelineMetrics.getMetrics().add(metric2);
- } else {
- LOG.warn("No test data name specified");
- }
-
- //Invoke method
- if (CollectionUtils.isNotEmpty(input.getMethods())) {
- RFunctionInvoker.setScriptsDir("/etc/ambari-metrics-collector/conf/R-scripts");
- for (String methodType : input.getMethods()) {
- ResultSet result = RFunctionInvoker.executeMethod(methodType, getAsDataSeries(metric1), getAsDataSeries(metric2), input.getMethodConfigs());
- TimelineMetric timelineMetric = getAsTimelineMetric(result, methodType, input, currentTime, hostname);
- if (timelineMetric != null) {
- timelineMetrics.getMetrics().add(timelineMetric);
- }
- }
- } else {
- LOG.warn("No anomaly method requested");
- }
-
- return timelineMetrics;
- }
-
-
- private static TimelineMetric getAsTimelineMetric(ResultSet result, String methodType, MetricAnomalyDetectorTestInput input, long currentTime, String hostname) {
-
- if (result == null) {
- return null;
- }
-
- TimelineMetric timelineMetric = new TimelineMetric();
- if (methodType.equals("tukeys") || methodType.equals("ema")) {
- timelineMetric.setMetricName(input.getTrainDataName() + "_" + input.getTestDataName() + "_" + methodType + "_" + currentTime);
- timelineMetric.setHostName(hostname);
- timelineMetric.setAppId(appId);
- timelineMetric.setInstanceId(null);
- timelineMetric.setStartTime(currentTime);
-
- TreeMap<Long, Double> metricValues = new TreeMap<>();
- if (result.resultset.size() > 0) {
- double[] ts = result.resultset.get(0);
- double[] metrics = result.resultset.get(1);
- for (int i = 0; i < ts.length; i++) {
- if (i == 0) {
- timelineMetric.setStartTime((long) ts[i]);
- }
- metricValues.put((long) ts[i], metrics[i]);
- }
- }
- timelineMetric.setMetricValues(metricValues);
- return timelineMetric;
- }
- return null;
- }
-
-
- private static TreeMap<Long, Double> getAsTimeSeries(long currentTime, double[] values) {
-
- long startTime = currentTime - (values.length - 1) * 60 * 1000;
- TreeMap<Long, Double> metricValues = new TreeMap<>();
-
- for (int i = 0; i < values.length; i++) {
- metricValues.put(startTime, values[i]);
- startTime += (60 * 1000);
- }
- return metricValues;
- }
-
- private static DataSeries getAsDataSeries(TimelineMetric timelineMetric) {
-
- TreeMap<Long, Double> metricValues = timelineMetric.getMetricValues();
- double[] timestamps = new double[metricValues.size()];
- double[] values = new double[metricValues.size()];
- int i = 0;
-
- for (Long timestamp : metricValues.keySet()) {
- timestamps[i] = timestamp;
- values[i++] = metricValues.get(timestamp);
- }
- return new DataSeries(timelineMetric.getMetricName() + "_" + timelineMetric.getAppId() + "_" + timelineMetric.getHostName(), timestamps, values);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java
deleted file mode 100644
index b817f3e..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.testing.utilities;
-
-/**
- * Class which was originally used to send test series from AMS to Spark through Kafka.
- */
-
-public class TestMetricSeriesGenerator {
- //implements Runnable {
-
-// private Map<TestSeriesInputRequest, AbstractMetricSeries> configuredSeries = new HashMap<>();
-// private static final Log LOG = LogFactory.getLog(TestMetricSeriesGenerator.class);
-// private TimelineMetricStore metricStore;
-// private String hostname;
-//
-// public TestMetricSeriesGenerator(TimelineMetricStore metricStore) {
-// this.metricStore = metricStore;
-// try {
-// this.hostname = InetAddress.getLocalHost().getHostName();
-// } catch (UnknownHostException e) {
-// e.printStackTrace();
-// }
-// }
-//
-// public void addSeries(TestSeriesInputRequest inputRequest) {
-// if (!configuredSeries.containsKey(inputRequest)) {
-// AbstractMetricSeries metricSeries = MetricSeriesGeneratorFactory.generateSeries(inputRequest.getSeriesType(), inputRequest.getConfigs());
-// configuredSeries.put(inputRequest, metricSeries);
-// LOG.info("Added series " + inputRequest.getSeriesName());
-// }
-// }
-//
-// public void removeSeries(String seriesName) {
-// boolean isPresent = false;
-// TestSeriesInputRequest tbd = null;
-// for (TestSeriesInputRequest inputRequest : configuredSeries.keySet()) {
-// if (inputRequest.getSeriesName().equals(seriesName)) {
-// isPresent = true;
-// tbd = inputRequest;
-// }
-// }
-// if (isPresent) {
-// LOG.info("Removing series " + seriesName);
-// configuredSeries.remove(tbd);
-// } else {
-// LOG.info("Series not found : " + seriesName);
-// }
-// }
-//
-// @Override
-// public void run() {
-// long currentTime = System.currentTimeMillis();
-// TimelineMetrics timelineMetrics = new TimelineMetrics();
-//
-// for (TestSeriesInputRequest input : configuredSeries.keySet()) {
-// AbstractMetricSeries metricSeries = configuredSeries.get(input);
-// TimelineMetric timelineMetric = new TimelineMetric();
-// timelineMetric.setMetricName(input.getSeriesName());
-// timelineMetric.setAppId("anomaly-engine-test-metric");
-// timelineMetric.setInstanceId(null);
-// timelineMetric.setStartTime(currentTime);
-// timelineMetric.setHostName(hostname);
-// TreeMap<Long, Double> metricValues = new TreeMap();
-// metricValues.put(currentTime, metricSeries.nextValue());
-// timelineMetric.setMetricValues(metricValues);
-// timelineMetrics.addOrMergeTimelineMetric(timelineMetric);
-// LOG.info("Emitting metric with appId = " + timelineMetric.getAppId());
-// }
-// try {
-// LOG.info("Publishing test metrics for " + timelineMetrics.getMetrics().size() + " series.");
-// metricStore.putMetrics(timelineMetrics);
-// } catch (Exception e) {
-// LOG.error(e);
-// }
-// }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java
deleted file mode 100644
index a424f8e..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.prototype.testing.utilities;
-
-import org.apache.htrace.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper;
-
-import javax.xml.bind.annotation.XmlRootElement;
-import java.util.Collections;
-import java.util.Map;
-
-@XmlRootElement
-public class TestSeriesInputRequest {
-
- private String seriesName;
- private String seriesType;
- private Map<String, String> configs;
-
- public TestSeriesInputRequest() {
- }
-
- public TestSeriesInputRequest(String seriesName, String seriesType, Map<String, String> configs) {
- this.seriesName = seriesName;
- this.seriesType = seriesType;
- this.configs = configs;
- }
-
- public String getSeriesName() {
- return seriesName;
- }
-
- public void setSeriesName(String seriesName) {
- this.seriesName = seriesName;
- }
-
- public String getSeriesType() {
- return seriesType;
- }
-
- public void setSeriesType(String seriesType) {
- this.seriesType = seriesType;
- }
-
- public Map<String, String> getConfigs() {
- return configs;
- }
-
- public void setConfigs(Map<String, String> configs) {
- this.configs = configs;
- }
-
- @Override
- public boolean equals(Object o) {
- TestSeriesInputRequest anotherInput = (TestSeriesInputRequest)o;
- return anotherInput.getSeriesName().equals(this.getSeriesName());
- }
-
- @Override
- public int hashCode() {
- return seriesName.hashCode();
- }
-
- public static void main(String[] args) {
-
- ObjectMapper objectMapper = new ObjectMapper();
- TestSeriesInputRequest testSeriesInputRequest = new TestSeriesInputRequest("test", "ema", Collections.singletonMap("key","value"));
- try {
- System.out.print(objectMapper.writeValueAsString(testSeriesInputRequest));
- } catch (JsonProcessingException e) {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java
deleted file mode 100644
index a8e31bf..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-public interface AbstractMetricSeries {
-
- public double nextValue();
- public double[] getSeries(int n);
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java
deleted file mode 100644
index 4158ff4..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class DualBandMetricSeries implements AbstractMetricSeries {
-
- double lowBandValue = 0.0;
- double lowBandDeviationPercentage = 0.0;
- int lowBandPeriodSize = 10;
- double highBandValue = 1.0;
- double highBandDeviationPercentage = 0.0;
- int highBandPeriodSize = 10;
-
- Random random = new Random();
- double lowBandValueLowerLimit, lowBandValueHigherLimit;
- double highBandLowerLimit, highBandUpperLimit;
- int l = 0, h = 0;
-
- public DualBandMetricSeries(double lowBandValue,
- double lowBandDeviationPercentage,
- int lowBandPeriodSize,
- double highBandValue,
- double highBandDeviationPercentage,
- int highBandPeriodSize) {
- this.lowBandValue = lowBandValue;
- this.lowBandDeviationPercentage = lowBandDeviationPercentage;
- this.lowBandPeriodSize = lowBandPeriodSize;
- this.highBandValue = highBandValue;
- this.highBandDeviationPercentage = highBandDeviationPercentage;
- this.highBandPeriodSize = highBandPeriodSize;
- init();
- }
-
- private void init() {
- lowBandValueLowerLimit = lowBandValue - lowBandDeviationPercentage * lowBandValue;
- lowBandValueHigherLimit = lowBandValue + lowBandDeviationPercentage * lowBandValue;
- highBandLowerLimit = highBandValue - highBandDeviationPercentage * highBandValue;
- highBandUpperLimit = highBandValue + highBandDeviationPercentage * highBandValue;
- }
-
- @Override
- public double nextValue() {
-
- double value = 0.0;
-
- if (l < lowBandPeriodSize) {
- value = lowBandValueLowerLimit + (lowBandValueHigherLimit - lowBandValueLowerLimit) * random.nextDouble();
- l++;
- } else if (h < highBandPeriodSize) {
- value = highBandLowerLimit + (highBandUpperLimit - highBandLowerLimit) * random.nextDouble();
- h++;
- }
-
- if (l == lowBandPeriodSize && h == highBandPeriodSize) {
- l = 0;
- h = 0;
- }
-
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
- double[] series = new double[n];
- for (int i = 0; i < n; i++) {
- series[i] = nextValue();
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java
deleted file mode 100644
index 1e37ff3..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Random;
-
-public class MetricSeriesGeneratorFactory {
-
- /**
- * Return a normally distributed data series with some deviation % and outliers.
- *
- * @param n size of the data series
- * @param value The value around which the uniform data series is centered on.
- * @param deviationPercentage The allowed deviation % on either side of the uniform value. For example, if value = 10, and deviation % is 0.1, the series values lie between 0.9 to 1.1.
- * @param outlierProbability The probability of finding an outlier in the series.
- * @param outlierDeviationLowerPercentage min percentage outlier should be away from the uniform value in % terms. if value = 10 and outlierDeviationPercentage = 30%, the outlier is 7 and 13.
- * @param outlierDeviationHigherPercentage max percentage outlier should be away from the uniform value in % terms. if value = 10 and outlierDeviationPercentage = 60%, the outlier is 4 and 16.
- * @param outliersAboveValue Outlier should be greater or smaller than the value.
- * @return uniform series
- */
- public static double[] createUniformSeries(int n,
- double value,
- double deviationPercentage,
- double outlierProbability,
- double outlierDeviationLowerPercentage,
- double outlierDeviationHigherPercentage,
- boolean outliersAboveValue) {
-
- UniformMetricSeries metricSeries = new UniformMetricSeries(value,
- deviationPercentage,
- outlierProbability,
- outlierDeviationLowerPercentage,
- outlierDeviationHigherPercentage,
- outliersAboveValue);
-
- return metricSeries.getSeries(n);
- }
-
-
- /**
- * /**
- * Returns a normally distributed series.
- *
- * @param n size of the data series
- * @param mean mean of the distribution
- * @param sd sd of the distribution
- * @param outlierProbability sd of the distribution
- * @param outlierDeviationSDTimesLower Lower Limit of the outlier with respect to times sdev from the mean.
- * @param outlierDeviationSDTimesHigher Higher Limit of the outlier with respect to times sdev from the mean.
- * @param outlierOnRightEnd Outlier should be on the right end or the left end.
- * @return normal series
- */
- public static double[] createNormalSeries(int n,
- double mean,
- double sd,
- double outlierProbability,
- double outlierDeviationSDTimesLower,
- double outlierDeviationSDTimesHigher,
- boolean outlierOnRightEnd) {
-
-
- NormalMetricSeries metricSeries = new NormalMetricSeries(mean,
- sd,
- outlierProbability,
- outlierDeviationSDTimesLower,
- outlierDeviationSDTimesHigher,
- outlierOnRightEnd);
-
- return metricSeries.getSeries(n);
- }
-
-
- /**
- * Returns a monotonically increasing / decreasing series
- *
- * @param n size of the data series
- * @param startValue Start value of the monotonic sequence
- * @param slope direction of monotonicity m > 0 for increasing and m < 0 for decreasing.
- * @param deviationPercentage The allowed deviation % on either side of the current 'y' value. For example, if current value = 10 according to slope, and deviation % is 0.1, the series values lie between 0.9 to 1.1.
- * @param outlierProbability The probability of finding an outlier in the series.
- * @param outlierDeviationLowerPercentage min percentage outlier should be away from the current 'y' value in % terms. if value = 10 and outlierDeviationPercentage = 30%, the outlier is 7 and 13.
- * @param outlierDeviationHigherPercentage max percentage outlier should be away from the current 'y' value in % terms. if value = 10 and outlierDeviationPercentage = 60%, the outlier is 4 and 16.
- * @param outliersAboveValue Outlier should be greater or smaller than the 'y' value.
- * @return
- */
- public static double[] createMonotonicSeries(int n,
- double startValue,
- double slope,
- double deviationPercentage,
- double outlierProbability,
- double outlierDeviationLowerPercentage,
- double outlierDeviationHigherPercentage,
- boolean outliersAboveValue) {
-
- MonotonicMetricSeries metricSeries = new MonotonicMetricSeries(startValue,
- slope,
- deviationPercentage,
- outlierProbability,
- outlierDeviationLowerPercentage,
- outlierDeviationHigherPercentage,
- outliersAboveValue);
-
- return metricSeries.getSeries(n);
- }
-
-
- /**
- * Returns a dual band series (lower and higher)
- *
- * @param n size of the data series
- * @param lowBandValue lower band value
- * @param lowBandDeviationPercentage lower band deviation
- * @param lowBandPeriodSize lower band
- * @param highBandValue high band centre value
- * @param highBandDeviationPercentage high band deviation.
- * @param highBandPeriodSize high band size
- * @return
- */
- public static double[] getDualBandSeries(int n,
- double lowBandValue,
- double lowBandDeviationPercentage,
- int lowBandPeriodSize,
- double highBandValue,
- double highBandDeviationPercentage,
- int highBandPeriodSize) {
-
- DualBandMetricSeries metricSeries = new DualBandMetricSeries(lowBandValue,
- lowBandDeviationPercentage,
- lowBandPeriodSize,
- highBandValue,
- highBandDeviationPercentage,
- highBandPeriodSize);
-
- return metricSeries.getSeries(n);
- }
-
- /**
- * Returns a step function series.
- *
- * @param n size of the data series
- * @param startValue start steady value
- * @param steadyValueDeviationPercentage required devation in the steady state value
- * @param steadyPeriodSlope direction of monotonicity m > 0 for increasing and m < 0 for decreasing, m = 0 no increase or decrease.
- * @param steadyPeriodMinSize min size for step period
- * @param steadyPeriodMaxSize max size for step period.
- * @param stepChangePercentage Increase / decrease in steady state to denote a step in terms of deviation percentage from the last value.
- * @param upwardStep upward or downward step.
- * @return
- */
- public static double[] getStepFunctionSeries(int n,
- double startValue,
- double steadyValueDeviationPercentage,
- double steadyPeriodSlope,
- int steadyPeriodMinSize,
- int steadyPeriodMaxSize,
- double stepChangePercentage,
- boolean upwardStep) {
-
- StepFunctionMetricSeries metricSeries = new StepFunctionMetricSeries(startValue,
- steadyValueDeviationPercentage,
- steadyPeriodSlope,
- steadyPeriodMinSize,
- steadyPeriodMaxSize,
- stepChangePercentage,
- upwardStep);
-
- return metricSeries.getSeries(n);
- }
-
- /**
- * Series with small period of turbulence and then back to steady.
- *
- * @param n size of the data series
- * @param steadyStateValue steady state center value
- * @param steadyStateDeviationPercentage steady state deviation in percentage
- * @param turbulentPeriodDeviationLowerPercentage turbulent state lower limit in terms of percentage from centre value.
- * @param turbulentPeriodDeviationHigherPercentage turbulent state higher limit in terms of percentage from centre value.
- * @param turbulentPeriodLength turbulent period length (number of points)
- * @param turbulentStatePosition Where the turbulent state should be 0 - at the beginning, 1 - in the middle (25% - 50% of the series), 2 - at the end of the series.
- * @return
- */
- public static double[] getSteadySeriesWithTurbulentPeriod(int n,
- double steadyStateValue,
- double steadyStateDeviationPercentage,
- double turbulentPeriodDeviationLowerPercentage,
- double turbulentPeriodDeviationHigherPercentage,
- int turbulentPeriodLength,
- int turbulentStatePosition
- ) {
-
-
- SteadyWithTurbulenceMetricSeries metricSeries = new SteadyWithTurbulenceMetricSeries(n,
- steadyStateValue,
- steadyStateDeviationPercentage,
- turbulentPeriodDeviationLowerPercentage,
- turbulentPeriodDeviationHigherPercentage,
- turbulentPeriodLength,
- turbulentStatePosition);
-
- return metricSeries.getSeries(n);
- }
-
-
- public static double[] generateSeries(String type, int n, Map<String, String> configs) {
-
- double[] series;
- switch (type) {
-
- case "normal":
- series = createNormalSeries(n,
- Double.parseDouble(configs.getOrDefault("mean", "0")),
- Double.parseDouble(configs.getOrDefault("sd", "1")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesLower", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesHigher", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outlierOnRightEnd", "true")));
- break;
-
- case "uniform":
- series = createUniformSeries(n,
- Double.parseDouble(configs.getOrDefault("value", "10")),
- Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
- break;
-
- case "monotonic":
- series = createMonotonicSeries(n,
- Double.parseDouble(configs.getOrDefault("startValue", "10")),
- Double.parseDouble(configs.getOrDefault("slope", "0")),
- Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
- break;
-
- case "dualband":
- series = getDualBandSeries(n,
- Double.parseDouble(configs.getOrDefault("lowBandValue", "10")),
- Double.parseDouble(configs.getOrDefault("lowBandDeviationPercentage", "0")),
- Integer.parseInt(configs.getOrDefault("lowBandPeriodSize", "0")),
- Double.parseDouble(configs.getOrDefault("highBandValue", "10")),
- Double.parseDouble(configs.getOrDefault("highBandDeviationPercentage", "0")),
- Integer.parseInt(configs.getOrDefault("highBandPeriodSize", "0")));
- break;
-
- case "step":
- series = getStepFunctionSeries(n,
- Double.parseDouble(configs.getOrDefault("startValue", "10")),
- Double.parseDouble(configs.getOrDefault("steadyValueDeviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("steadyPeriodSlope", "0")),
- Integer.parseInt(configs.getOrDefault("steadyPeriodMinSize", "0")),
- Integer.parseInt(configs.getOrDefault("steadyPeriodMaxSize", "0")),
- Double.parseDouble(configs.getOrDefault("stepChangePercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("upwardStep", "true")));
- break;
-
- case "turbulence":
- series = getSteadySeriesWithTurbulentPeriod(n,
- Double.parseDouble(configs.getOrDefault("steadyStateValue", "10")),
- Double.parseDouble(configs.getOrDefault("steadyStateDeviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationHigherPercentage", "10")),
- Integer.parseInt(configs.getOrDefault("turbulentPeriodLength", "0")),
- Integer.parseInt(configs.getOrDefault("turbulentStatePosition", "0")));
- break;
-
- default:
- series = createNormalSeries(n,
- 0,
- 1,
- 0,
- 0,
- 0,
- true);
- }
- return series;
- }
-
- public static AbstractMetricSeries generateSeries(String type, Map<String, String> configs) {
-
- AbstractMetricSeries series;
- switch (type) {
-
- case "normal":
- series = new NormalMetricSeries(Double.parseDouble(configs.getOrDefault("mean", "0")),
- Double.parseDouble(configs.getOrDefault("sd", "1")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesLower", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesHigher", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outlierOnRightEnd", "true")));
- break;
-
- case "uniform":
- series = new UniformMetricSeries(
- Double.parseDouble(configs.getOrDefault("value", "10")),
- Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
- break;
-
- case "monotonic":
- series = new MonotonicMetricSeries(
- Double.parseDouble(configs.getOrDefault("startValue", "10")),
- Double.parseDouble(configs.getOrDefault("slope", "0")),
- Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierProbability", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true")));
- break;
-
- case "dualband":
- series = new DualBandMetricSeries(
- Double.parseDouble(configs.getOrDefault("lowBandValue", "10")),
- Double.parseDouble(configs.getOrDefault("lowBandDeviationPercentage", "0")),
- Integer.parseInt(configs.getOrDefault("lowBandPeriodSize", "0")),
- Double.parseDouble(configs.getOrDefault("highBandValue", "10")),
- Double.parseDouble(configs.getOrDefault("highBandDeviationPercentage", "0")),
- Integer.parseInt(configs.getOrDefault("highBandPeriodSize", "0")));
- break;
-
- case "step":
- series = new StepFunctionMetricSeries(
- Double.parseDouble(configs.getOrDefault("startValue", "10")),
- Double.parseDouble(configs.getOrDefault("steadyValueDeviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("steadyPeriodSlope", "0")),
- Integer.parseInt(configs.getOrDefault("steadyPeriodMinSize", "0")),
- Integer.parseInt(configs.getOrDefault("steadyPeriodMaxSize", "0")),
- Double.parseDouble(configs.getOrDefault("stepChangePercentage", "0")),
- Boolean.parseBoolean(configs.getOrDefault("upwardStep", "true")));
- break;
-
- case "turbulence":
- series = new SteadyWithTurbulenceMetricSeries(
- Integer.parseInt(configs.getOrDefault("approxSeriesLength", "100")),
- Double.parseDouble(configs.getOrDefault("steadyStateValue", "10")),
- Double.parseDouble(configs.getOrDefault("steadyStateDeviationPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationLowerPercentage", "0")),
- Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationHigherPercentage", "10")),
- Integer.parseInt(configs.getOrDefault("turbulentPeriodLength", "0")),
- Integer.parseInt(configs.getOrDefault("turbulentStatePosition", "0")));
- break;
-
- default:
- series = new NormalMetricSeries(0,
- 1,
- 0,
- 0,
- 0,
- true);
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java
deleted file mode 100644
index a883d08..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class MonotonicMetricSeries implements AbstractMetricSeries {
-
- double startValue = 0.0;
- double slope = 0.5;
- double deviationPercentage = 0.0;
- double outlierProbability = 0.0;
- double outlierDeviationLowerPercentage = 0.0;
- double outlierDeviationHigherPercentage = 0.0;
- boolean outliersAboveValue = true;
-
- Random random = new Random();
- double nonOutlierProbability;
-
- // y = mx + c
- double y;
- double m;
- double x;
- double c;
-
- public MonotonicMetricSeries(double startValue,
- double slope,
- double deviationPercentage,
- double outlierProbability,
- double outlierDeviationLowerPercentage,
- double outlierDeviationHigherPercentage,
- boolean outliersAboveValue) {
- this.startValue = startValue;
- this.slope = slope;
- this.deviationPercentage = deviationPercentage;
- this.outlierProbability = outlierProbability;
- this.outlierDeviationLowerPercentage = outlierDeviationLowerPercentage;
- this.outlierDeviationHigherPercentage = outlierDeviationHigherPercentage;
- this.outliersAboveValue = outliersAboveValue;
- init();
- }
-
- private void init() {
- y = startValue;
- m = slope;
- x = 1;
- c = y - (m * x);
- nonOutlierProbability = 1.0 - outlierProbability;
- }
-
- @Override
- public double nextValue() {
-
- double value;
- double probability = random.nextDouble();
-
- y = m * x + c;
- if (probability <= nonOutlierProbability) {
- double valueDeviationLowerLimit = y - deviationPercentage * y;
- double valueDeviationHigherLimit = y + deviationPercentage * y;
- value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
- } else {
- if (outliersAboveValue) {
- double outlierLowerLimit = y + outlierDeviationLowerPercentage * y;
- double outlierUpperLimit = y + outlierDeviationHigherPercentage * y;
- value = outlierLowerLimit + (outlierUpperLimit - outlierLowerLimit) * random.nextDouble();
- } else {
- double outlierLowerLimit = y - outlierDeviationLowerPercentage * y;
- double outlierUpperLimit = y - outlierDeviationHigherPercentage * y;
- value = outlierUpperLimit + (outlierLowerLimit - outlierUpperLimit) * random.nextDouble();
- }
- }
- x++;
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
- double[] series = new double[n];
- for (int i = 0; i < n; i++) {
- series[i] = nextValue();
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java
deleted file mode 100644
index cc83d2c..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class NormalMetricSeries implements AbstractMetricSeries {
-
- double mean = 0.0;
- double sd = 1.0;
- double outlierProbability = 0.0;
- double outlierDeviationSDTimesLower = 0.0;
- double outlierDeviationSDTimesHigher = 0.0;
- boolean outlierOnRightEnd = true;
-
- Random random = new Random();
- double nonOutlierProbability;
-
-
- public NormalMetricSeries(double mean,
- double sd,
- double outlierProbability,
- double outlierDeviationSDTimesLower,
- double outlierDeviationSDTimesHigher,
- boolean outlierOnRightEnd) {
- this.mean = mean;
- this.sd = sd;
- this.outlierProbability = outlierProbability;
- this.outlierDeviationSDTimesLower = outlierDeviationSDTimesLower;
- this.outlierDeviationSDTimesHigher = outlierDeviationSDTimesHigher;
- this.outlierOnRightEnd = outlierOnRightEnd;
- init();
- }
-
- private void init() {
- nonOutlierProbability = 1.0 - outlierProbability;
- }
-
- @Override
- public double nextValue() {
-
- double value;
- double probability = random.nextDouble();
-
- if (probability <= nonOutlierProbability) {
- value = random.nextGaussian() * sd + mean;
- } else {
- if (outlierOnRightEnd) {
- value = mean + (outlierDeviationSDTimesLower + (outlierDeviationSDTimesHigher - outlierDeviationSDTimesLower) * random.nextDouble()) * sd;
- } else {
- value = mean - (outlierDeviationSDTimesLower + (outlierDeviationSDTimesHigher - outlierDeviationSDTimesLower) * random.nextDouble()) * sd;
- }
- }
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
- double[] series = new double[n];
- for (int i = 0; i < n; i++) {
- series[i] = nextValue();
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java
deleted file mode 100644
index c4ed3ba..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class SteadyWithTurbulenceMetricSeries implements AbstractMetricSeries {
-
- double steadyStateValue = 0.0;
- double steadyStateDeviationPercentage = 0.0;
- double turbulentPeriodDeviationLowerPercentage = 0.3;
- double turbulentPeriodDeviationHigherPercentage = 0.5;
- int turbulentPeriodLength = 5;
- int turbulentStatePosition = 1;
- int approximateSeriesLength = 10;
-
- Random random = new Random();
- double valueDeviationLowerLimit;
- double valueDeviationHigherLimit;
- double tPeriodLowerLimit;
- double tPeriodUpperLimit;
- int tPeriodStartIndex = 0;
- int index = 0;
-
- public SteadyWithTurbulenceMetricSeries(int approximateSeriesLength,
- double steadyStateValue,
- double steadyStateDeviationPercentage,
- double turbulentPeriodDeviationLowerPercentage,
- double turbulentPeriodDeviationHigherPercentage,
- int turbulentPeriodLength,
- int turbulentStatePosition) {
- this.approximateSeriesLength = approximateSeriesLength;
- this.steadyStateValue = steadyStateValue;
- this.steadyStateDeviationPercentage = steadyStateDeviationPercentage;
- this.turbulentPeriodDeviationLowerPercentage = turbulentPeriodDeviationLowerPercentage;
- this.turbulentPeriodDeviationHigherPercentage = turbulentPeriodDeviationHigherPercentage;
- this.turbulentPeriodLength = turbulentPeriodLength;
- this.turbulentStatePosition = turbulentStatePosition;
- init();
- }
-
- private void init() {
-
- if (turbulentStatePosition == 1) {
- tPeriodStartIndex = (int) (0.25 * approximateSeriesLength + (0.25 * approximateSeriesLength * random.nextDouble()));
- } else if (turbulentStatePosition == 2) {
- tPeriodStartIndex = approximateSeriesLength - turbulentPeriodLength;
- }
-
- valueDeviationLowerLimit = steadyStateValue - steadyStateDeviationPercentage * steadyStateValue;
- valueDeviationHigherLimit = steadyStateValue + steadyStateDeviationPercentage * steadyStateValue;
-
- tPeriodLowerLimit = steadyStateValue + turbulentPeriodDeviationLowerPercentage * steadyStateValue;
- tPeriodUpperLimit = steadyStateValue + turbulentPeriodDeviationHigherPercentage * steadyStateValue;
- }
-
- @Override
- public double nextValue() {
-
- double value;
-
- if (index >= tPeriodStartIndex && index <= (tPeriodStartIndex + turbulentPeriodLength)) {
- value = tPeriodLowerLimit + (tPeriodUpperLimit - tPeriodLowerLimit) * random.nextDouble();
- } else {
- value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
- }
- index++;
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
-
- double[] series = new double[n];
- int turbulentPeriodStartIndex = 0;
-
- if (turbulentStatePosition == 1) {
- turbulentPeriodStartIndex = (int) (0.25 * n + (0.25 * n * random.nextDouble()));
- } else if (turbulentStatePosition == 2) {
- turbulentPeriodStartIndex = n - turbulentPeriodLength;
- }
-
- double valueDevLowerLimit = steadyStateValue - steadyStateDeviationPercentage * steadyStateValue;
- double valueDevHigherLimit = steadyStateValue + steadyStateDeviationPercentage * steadyStateValue;
-
- double turbulentPeriodLowerLimit = steadyStateValue + turbulentPeriodDeviationLowerPercentage * steadyStateValue;
- double turbulentPeriodUpperLimit = steadyStateValue + turbulentPeriodDeviationHigherPercentage * steadyStateValue;
-
- for (int i = 0; i < n; i++) {
- if (i >= turbulentPeriodStartIndex && i < (turbulentPeriodStartIndex + turbulentPeriodLength)) {
- series[i] = turbulentPeriodLowerLimit + (turbulentPeriodUpperLimit - turbulentPeriodLowerLimit) * random.nextDouble();
- } else {
- series[i] = valueDevLowerLimit + (valueDevHigherLimit - valueDevLowerLimit) * random.nextDouble();
- }
- }
-
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java
deleted file mode 100644
index d5beb48..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class StepFunctionMetricSeries implements AbstractMetricSeries {
-
- double startValue = 0.0;
- double steadyValueDeviationPercentage = 0.0;
- double steadyPeriodSlope = 0.5;
- int steadyPeriodMinSize = 10;
- int steadyPeriodMaxSize = 20;
- double stepChangePercentage = 0.0;
- boolean upwardStep = true;
-
- Random random = new Random();
-
- // y = mx + c
- double y;
- double m;
- double x;
- double c;
- int currentStepSize;
- int currentIndex;
-
- public StepFunctionMetricSeries(double startValue,
- double steadyValueDeviationPercentage,
- double steadyPeriodSlope,
- int steadyPeriodMinSize,
- int steadyPeriodMaxSize,
- double stepChangePercentage,
- boolean upwardStep) {
- this.startValue = startValue;
- this.steadyValueDeviationPercentage = steadyValueDeviationPercentage;
- this.steadyPeriodSlope = steadyPeriodSlope;
- this.steadyPeriodMinSize = steadyPeriodMinSize;
- this.steadyPeriodMaxSize = steadyPeriodMaxSize;
- this.stepChangePercentage = stepChangePercentage;
- this.upwardStep = upwardStep;
- init();
- }
-
- private void init() {
- y = startValue;
- m = steadyPeriodSlope;
- x = 1;
- c = y - (m * x);
-
- currentStepSize = (int) (steadyPeriodMinSize + (steadyPeriodMaxSize - steadyPeriodMinSize) * random.nextDouble());
- currentIndex = 0;
- }
-
- @Override
- public double nextValue() {
-
- double value = 0.0;
-
- if (currentIndex < currentStepSize) {
- y = m * x + c;
- double valueDeviationLowerLimit = y - steadyValueDeviationPercentage * y;
- double valueDeviationHigherLimit = y + steadyValueDeviationPercentage * y;
- value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
- x++;
- currentIndex++;
- }
-
- if (currentIndex == currentStepSize) {
- currentIndex = 0;
- currentStepSize = (int) (steadyPeriodMinSize + (steadyPeriodMaxSize - steadyPeriodMinSize) * random.nextDouble());
- if (upwardStep) {
- y = y + stepChangePercentage * y;
- } else {
- y = y - stepChangePercentage * y;
- }
- x = 1;
- c = y - (m * x);
- }
-
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
- double[] series = new double[n];
- for (int i = 0; i < n; i++) {
- series[i] = nextValue();
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java
deleted file mode 100644
index a2b0eea..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.alertservice.seriesgenerator;
-
-import java.util.Random;
-
-public class UniformMetricSeries implements AbstractMetricSeries {
-
- double value = 0.0;
- double deviationPercentage = 0.0;
- double outlierProbability = 0.0;
- double outlierDeviationLowerPercentage = 0.0;
- double outlierDeviationHigherPercentage = 0.0;
- boolean outliersAboveValue= true;
-
- Random random = new Random();
- double valueDeviationLowerLimit;
- double valueDeviationHigherLimit;
- double outlierLeftLowerLimit;
- double outlierLeftHigherLimit;
- double outlierRightLowerLimit;
- double outlierRightUpperLimit;
- double nonOutlierProbability;
-
-
- public UniformMetricSeries(double value,
- double deviationPercentage,
- double outlierProbability,
- double outlierDeviationLowerPercentage,
- double outlierDeviationHigherPercentage,
- boolean outliersAboveValue) {
- this.value = value;
- this.deviationPercentage = deviationPercentage;
- this.outlierProbability = outlierProbability;
- this.outlierDeviationLowerPercentage = outlierDeviationLowerPercentage;
- this.outlierDeviationHigherPercentage = outlierDeviationHigherPercentage;
- this.outliersAboveValue = outliersAboveValue;
- init();
- }
-
- private void init() {
- valueDeviationLowerLimit = value - deviationPercentage * value;
- valueDeviationHigherLimit = value + deviationPercentage * value;
-
- outlierLeftLowerLimit = value - outlierDeviationHigherPercentage * value;
- outlierLeftHigherLimit = value - outlierDeviationLowerPercentage * value;
- outlierRightLowerLimit = value + outlierDeviationLowerPercentage * value;
- outlierRightUpperLimit = value + outlierDeviationHigherPercentage * value;
-
- nonOutlierProbability = 1.0 - outlierProbability;
- }
-
- @Override
- public double nextValue() {
-
- double value;
- double probability = random.nextDouble();
-
- if (probability <= nonOutlierProbability) {
- value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble();
- } else {
- if (!outliersAboveValue) {
- value = outlierLeftLowerLimit + (outlierLeftHigherLimit - outlierLeftLowerLimit) * random.nextDouble();
- } else {
- value = outlierRightLowerLimit + (outlierRightUpperLimit - outlierRightLowerLimit) * random.nextDouble();
- }
- }
- return value;
- }
-
- @Override
- public double[] getSeries(int n) {
- double[] series = new double[n];
- for (int i = 0; i < n; i++) {
- series[i] = nextValue();
- }
- return series;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R
deleted file mode 100644
index 0b66095..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R
+++ /dev/null
@@ -1,96 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# EMA <- w * EMA + (1 - w) * x
-# EMS <- sqrt( w * EMS^2 + (1 - w) * (x - EMA)^2 )
-# Alarm = abs(x - EMA) > n * EMS
-
-ema_global <- function(train_data, test_data, w, n) {
-
-# res <- get_data(url)
-# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
-# names(data) <- c("TS", res$metrics[[1]]$metricname)
-# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2]
-# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
-
- anomalies <- data.frame()
- ema <- 0
- ems <- 0
-
- #Train Step
- for (x in train_data) {
- ema <- w*ema + (1-w)*x
- ems <- sqrt(w* ems^2 + (1 - w)*(x - ema)^2)
- }
-
- for ( i in 1:length(test_data[,1])) {
- x <- test_data[i,2]
- if (abs(x - ema) > n*ems) {
- anomaly <- c(as.numeric(test_data[i,1]), x)
- # print (anomaly)
- anomalies <- rbind(anomalies, anomaly)
- }
- ema <- w*ema + (1-w)*x
- ems <- sqrt(w* ems^2 + (1 - w)*(x - ema)^2)
- }
-
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS", "Value")
- }
- return (anomalies)
-}
-
-ema_daily <- function(train_data, test_data, w, n) {
-
-# res <- get_data(url)
-# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
-# names(data) <- c("TS", res$metrics[[1]]$metricname)
-# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), ]
-# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
-
- anomalies <- data.frame()
- ema <- vector("numeric", 7)
- ems <- vector("numeric", 7)
-
- #Train Step
- for ( i in 1:length(train_data[,1])) {
- x <- train_data[i,2]
- time <- as.POSIXlt(as.numeric(train_data[i,1])/1000, origin = "1970-01-01" ,tz = "GMT")
- index <- time$wday
- ema[index] <- w*ema[index] + (1-w)*x
- ems[index] <- sqrt(w* ems[index]^2 + (1 - w)*(x - ema[index])^2)
- }
-
- for ( i in 1:length(test_data[,1])) {
- x <- test_data[i,2]
- time <- as.POSIXlt(as.numeric(test_data[i,1])/1000, origin = "1970-01-01" ,tz = "GMT")
- index <- time$wday
-
- if (abs(x - ema[index+1]) > n*ems[index+1]) {
- anomaly <- c(as.numeric(test_data[i,1]), x)
- # print (anomaly)
- anomalies <- rbind(anomalies, anomaly)
- }
- ema[index+1] <- w*ema[index+1] + (1-w)*x
- ems[index+1] <- sqrt(w* ems[index+1]^2 + (1 - w)*(x - ema[index+1])^2)
- }
-
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS", "Value")
- }
- return(anomalies)
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r
deleted file mode 100644
index bca3366..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r
+++ /dev/null
@@ -1,67 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-hsdev_daily <- function(train_data, test_data, n, num_historic_periods, interval, period) {
-
- #res <- get_data(url)
- #data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
- #names(data) <- c("TS", res$metrics[[1]]$metricname)
- anomalies <- data.frame()
-
- granularity <- train_data[2,1] - train_data[1,1]
- test_start <- test_data[1,1]
- test_end <- test_data[length(test_data[1,]),1]
- train_start <- test_start - num_historic_periods*period
- # round to start of day
- train_start <- train_start - (train_start %% interval)
-
- time <- as.POSIXlt(as.numeric(test_data[1,1])/1000, origin = "1970-01-01" ,tz = "GMT")
- test_data_day <- time$wday
-
- h_data <- c()
- for ( i in length(train_data[,1]):1) {
- ts <- train_data[i,1]
- if ( ts < train_start) {
- break
- }
- time <- as.POSIXlt(as.numeric(ts)/1000, origin = "1970-01-01" ,tz = "GMT")
- if (time$wday == test_data_day) {
- x <- train_data[i,2]
- h_data <- c(h_data, x)
- }
- }
-
- if (length(h_data) < 2*length(test_data[,1])) {
- cat ("\nNot enough training data")
- return (anomalies)
- }
-
- past_median <- median(h_data)
- past_sd <- sd(h_data)
- curr_median <- median(test_data[,2])
-
- if (abs(curr_median - past_median) > n * past_sd) {
- anomaly <- c(test_start, test_end, curr_median, past_median, past_sd)
- anomalies <- rbind(anomalies, anomaly)
- }
-
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS Start", "TS End", "Current Median", "Past Median", "Past SD")
- }
-
- return (anomalies)
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R
deleted file mode 100644
index 8956400..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ams_iforest <- function(url, train_start, train_end, test_start, test_end, threshold_score) {
-
- res <- get_data(url)
- num_metrics <- length(res$metrics)
- anomalies <- data.frame()
-
- metricname <- res$metrics[[1]]$metricname
- data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
- names(data) <- c("TS", res$metrics[[1]]$metricname)
-
- for (i in 2:num_metrics) {
- metricname <- res$metrics[[i]]$metricname
- df <- data.frame(as.numeric(names(res$metrics[[i]]$metrics)), as.numeric(res$metrics[[i]]$metrics))
- names(df) <- c("TS", res$metrics[[i]]$metricname)
- data <- merge(data, df)
- }
-
- algo_data <- data[ which(df$TS >= train_start & df$TS <= train_end) , ][c(1:num_metrics+1)]
- iForest <- IsolationTrees(algo_data)
- test_data <- data[ which(df$TS >= test_start & df$TS <= test_end) , ]
-
- if_res <- AnomalyScore(test_data[c(1:num_metrics+1)], iForest)
- for (i in 1:length(if_res$outF)) {
- index <- test_start+i-1
- if (if_res$outF[i] > threshold_score) {
- anomaly <- c(test_data[i,1], if_res$outF[i], if_res$pathLength[i])
- anomalies <- rbind(anomalies, anomaly)
- }
- }
-
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS", "Anomaly Score", "Path length")
- }
- return (anomalies)
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r
deleted file mode 100644
index f22bc15..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ams_ks <- function(train_data, test_data, p_value) {
-
-# res <- get_data(url)
-# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
-# names(data) <- c("TS", res$metrics[[1]]$metricname)
-# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2]
-# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), 2]
-
- anomalies <- data.frame()
- res <- ks.test(train_data[,2], test_data[,2])
-
- if (res[2] < p_value) {
- anomaly <- c(test_data[1,1], test_data[length(test_data),1], res[1], res[2])
- anomalies <- rbind(anomalies, anomaly)
- }
-
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS Start", "TS End", "D", "p-value")
- }
- return (anomalies)
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R
deleted file mode 100644
index 7650356..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R
+++ /dev/null
@@ -1,85 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-tukeys_anomalies <- data.frame()
-ema_global_anomalies <- data.frame()
-ema_daily_anomalies <- data.frame()
-ks_anomalies <- data.frame()
-hsdev_anomalies <- data.frame()
-
-init <- function() {
- tukeys_anomalies <- data.frame()
- ema_global_anomalies <- data.frame()
- ema_daily_anomalies <- data.frame()
- ks_anomalies <- data.frame()
- hsdev_anomalies <- data.frame()
-}
-
-test_methods <- function(data) {
-
- init()
- #res <- get_data(url)
- #data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
- #names(data) <- c("TS", res$metrics[[1]]$metricname)
-
- limit <- data[length(data[,1]),1]
- step <- data[2,1] - data[1,1]
-
- train_start <- data[1,1]
- train_end <- get_next_day_boundary(train_start, step, limit)
- test_start <- train_end + step
- test_end <- get_next_day_boundary(test_start, step, limit)
- i <- 1
- day <- 24*60*60*1000
-
- while (test_start < limit) {
-
- print (i)
- i <- i + 1
- train_data <- data[which(data$TS >= train_start & data$TS <= train_end),]
- test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
-
- #tukeys_anomalies <<- rbind(tukeys_anomalies, ams_tukeys(train_data, test_data, 3))
- #ema_global_anomalies <<- rbind(ema_global_anomalies, ema_global(train_data, test_data, 0.9, 3))
- #ema_daily_anomalies <<- rbind(ema_daily_anomalies, ema_daily(train_data, test_data, 0.9, 3))
- #ks_anomalies <<- rbind(ks_anomalies, ams_ks(train_data, test_data, 0.05))
- hsdev_train_data <- data[which(data$TS < test_start),]
- hsdev_anomalies <<- rbind(hsdev_anomalies, hsdev_daily(hsdev_train_data, test_data, 3, 3, day, 7*day))
-
- train_start <- test_start
- train_end <- get_next_day_boundary(train_start, step, limit)
- test_start <- train_end + step
- test_end <- get_next_day_boundary(test_start, step, limit)
- }
- return (hsdev_anomalies)
-}
-
-get_next_day_boundary <- function(start, step, limit) {
-
- if (start > limit) {
- return (-1)
- }
-
- while (start <= limit) {
- if (((start %% (24*60*60*1000)) - 28800000) == 0) {
- return (start)
- }
- start <- start + step
- }
- return (start)
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r
deleted file mode 100644
index 0312226..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r
+++ /dev/null
@@ -1,51 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-ams_tukeys <- function(train_data, test_data, n) {
-
-# res <- get_data(url)
-# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics))
-# names(data) <- c("TS", res$metrics[[1]]$metricname)
-# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2]
-# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ]
-
- anomalies <- data.frame()
- quantiles <- quantile(train_data[,2])
- iqr <- quantiles[4] - quantiles[2]
- niqr <- 0
-
- for ( i in 1:length(test_data[,1])) {
- x <- test_data[i,2]
- lb <- quantiles[2] - n*iqr
- ub <- quantiles[4] + n*iqr
- if ( (x < lb) || (x > ub) ) {
- if (iqr != 0) {
- if (x < lb) {
- niqr <- (quantiles[2] - x) / iqr
- } else {
- niqr <- (x - quantiles[4]) / iqr
- }
- }
- anomaly <- c(test_data[i,1], x, niqr)
- anomalies <- rbind(anomalies, anomaly)
- }
- }
- if(length(anomalies) > 0) {
- names(anomalies) <- c("TS", "Value", "niqr")
- }
- return (anomalies)
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties
deleted file mode 100644
index ab106c4..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-appIds=HOST
-
-collectorHost=localhost
-collectorPort=6188
-collectorProtocol=http
-
-zkQuorum=localhost:2181
-
-ambariServerHost=localhost
-clusterName=c1
-
-emaW=0.8
-emaN=3
-tukeysN=3
-pointInTimeTestInterval=300000
-pointInTimeTrainInterval=900000
-
-ksTestInterval=600000
-ksTrainInterval=600000
-hsdevNhp=3
-hsdevInterval=1800000;
-
-skipMetricPatterns=sdisk*,cpu_sintr*,proc*,disk*,boottime
-hosts=avijayan-ad-1.openstacklocal
\ No newline at end of file