You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2018/01/02 21:59:45 UTC
[7/7] ambari git commit: AMBARI-22717 : Remove Anomaly Detection code
from branch-3.0-ams. (avijayan)
AMBARI-22717 : Remove Anomaly Detection code from branch-3.0-ams. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/25c18121
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/25c18121
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/25c18121
Branch: refs/heads/branch-3.0-ams
Commit: 25c181210af69999cf58f0d60d2c9e07229fef9b
Parents: 8329f46
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Tue Jan 2 11:34:16 2018 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Tue Jan 2 12:15:48 2018 -0800
----------------------------------------------------------------------
.../libraries/functions/package_conditions.py | 4 -
.../conf/unix/ambari-metrics-admanager | 227 --------
.../conf/unix/config.yaml | 45 --
.../conf/unix/log4j.properties | 31 --
.../pom.xml | 528 -------------------
.../src/main/assemblies/empty.xml | 21 -
.../adservice/prototype/common/DataSeries.java | 38 --
.../adservice/prototype/common/ResultSet.java | 43 --
.../prototype/common/StatisticUtils.java | 59 ---
.../prototype/core/AmbariServerInterface.java | 119 -----
.../prototype/core/MetricKafkaProducer.java | 56 --
.../prototype/core/MetricSparkConsumer.java | 244 ---------
.../core/MetricsCollectorInterface.java | 237 ---------
.../prototype/core/PointInTimeADSystem.java | 260 ---------
.../prototype/core/RFunctionInvoker.java | 222 --------
.../adservice/prototype/core/TrendADSystem.java | 317 -----------
.../adservice/prototype/core/TrendMetric.java | 33 --
.../methods/AnomalyDetectionTechnique.java | 30 --
.../prototype/methods/MetricAnomaly.java | 84 ---
.../prototype/methods/ema/EmaModel.java | 131 -----
.../prototype/methods/ema/EmaModelLoader.java | 40 --
.../prototype/methods/ema/EmaTechnique.java | 151 ------
.../prototype/methods/hsdev/HsdevTechnique.java | 82 ---
.../prototype/methods/kstest/KSTechnique.java | 101 ----
.../MetricAnomalyDetectorTestInput.java | 126 -----
.../testing/utilities/MetricAnomalyTester.java | 151 ------
.../utilities/TestMetricSeriesGenerator.java | 92 ----
.../utilities/TestSeriesInputRequest.java | 88 ----
.../src/main/resources/R-scripts/ema.R | 96 ----
.../src/main/resources/R-scripts/hsdev.r | 67 ---
.../src/main/resources/R-scripts/iforest.R | 52 --
.../src/main/resources/R-scripts/kstest.r | 38 --
.../src/main/resources/R-scripts/test.R | 85 ---
.../src/main/resources/R-scripts/tukeys.r | 51 --
.../src/main/resources/hbase-site.xml | 286 ----------
.../src/main/resources/input-config.properties | 42 --
.../adservice/app/ADServiceScalaModule.scala | 50 --
.../adservice/app/AnomalyDetectionApp.scala | 80 ---
.../app/AnomalyDetectionAppConfig.scala | 89 ----
.../app/AnomalyDetectionAppModule.scala | 47 --
.../adservice/app/DefaultHealthCheck.scala | 25 -
.../metrics/adservice/app/GuiceInjector.scala | 56 --
.../configuration/AdServiceConfiguration.scala | 40 --
.../configuration/HBaseConfiguration.scala | 59 ---
.../MetricCollectorConfiguration.scala | 54 --
.../MetricDefinitionDBConfiguration.scala | 40 --
.../MetricDefinitionServiceConfiguration.scala | 31 --
.../configuration/SparkConfiguration.scala | 39 --
.../adservice/db/AdAnomalyStoreAccessor.scala | 36 --
.../adservice/db/AdMetadataStoreAccessor.scala | 53 --
.../db/AdMetadataStoreAccessorImpl.scala | 96 ----
.../adservice/db/AdMetadataStoreConstants.scala | 39 --
.../adservice/db/ConnectionProvider.scala | 45 --
.../adservice/db/DefaultPhoenixDataSource.scala | 79 ---
.../adservice/db/MetadataDatasource.scala | 79 ---
.../db/PhoenixAnomalyStoreAccessor.scala | 195 -------
.../db/PhoenixConnectionProvider.scala | 66 ---
.../adservice/db/PhoenixQueryConstants.scala | 108 ----
.../adservice/leveldb/LevelDBDatasource.scala | 128 -----
.../adservice/metadata/ADMetadataProvider.scala | 147 ------
.../metadata/InputMetricDefinitionParser.scala | 58 --
.../adservice/metadata/MetricDefinition.scala | 105 ----
.../metadata/MetricDefinitionService.scala | 78 ---
.../metadata/MetricDefinitionServiceImpl.scala | 242 ---------
.../metrics/adservice/metadata/MetricKey.scala | 53 --
.../metadata/MetricMetadataProvider.scala | 31 --
.../metadata/MetricSourceDefinition.scala | 85 ---
.../metadata/MetricSourceDefinitionType.scala | 26 -
.../model/AnomalyDetectionMethod.scala | 23 -
.../metrics/adservice/model/AnomalyType.scala | 26 -
.../adservice/model/MetricAnomalyInstance.scala | 32 --
.../model/PointInTimeAnomalyInstance.scala | 46 --
.../ambari/metrics/adservice/model/Range.scala | 44 --
.../ambari/metrics/adservice/model/Season.scala | 122 -----
.../metrics/adservice/model/SeasonType.scala | 24 -
.../metrics/adservice/model/TimeRange.scala | 39 --
.../adservice/model/TrendAnomalyInstance.scala | 44 --
.../adservice/resource/AnomalyResource.scala | 80 ---
.../resource/MetricDefinitionResource.scala | 109 ----
.../adservice/resource/RootResource.scala | 38 --
.../adservice/resource/SubsystemResource.scala | 26 -
.../adservice/service/ADQueryService.scala | 34 --
.../adservice/service/ADQueryServiceImpl.scala | 56 --
.../adservice/service/AbstractADService.scala | 44 --
.../spark/prototype/MetricAnomalyDetector.scala | 110 ----
.../spark/prototype/SparkPhoenixReader.scala | 73 ---
.../adservice/prototype/TestEmaTechnique.java | 106 ----
.../prototype/TestRFunctionInvoker.java | 161 ------
.../metrics/adservice/prototype/TestTukeys.java | 100 ----
.../seriesgenerator/AbstractMetricSeries.java | 25 -
.../seriesgenerator/DualBandMetricSeries.java | 88 ----
.../MetricSeriesGeneratorFactory.java | 377 -------------
.../MetricSeriesGeneratorTest.java | 101 ----
.../seriesgenerator/MonotonicMetricSeries.java | 101 ----
.../seriesgenerator/NormalMetricSeries.java | 81 ---
.../SteadyWithTurbulenceMetricSeries.java | 115 ----
.../StepFunctionMetricSeries.java | 107 ----
.../seriesgenerator/UniformMetricSeries.java | 95 ----
.../src/test/resources/config.yaml | 35 --
.../app/AnomalyDetectionAppConfigTest.scala | 67 ---
.../app/DefaultADResourceSpecTest.scala | 57 --
.../adservice/app/DropwizardAppRuleHelper.scala | 39 --
.../app/DropwizardResourceTestRuleHelper.scala | 33 --
.../db/PhoenixAnomalyStoreAccessorTest.scala | 26 -
.../leveldb/LevelDBDataSourceTest.scala | 57 --
.../metadata/AMSMetadataProviderTest.scala | 49 --
.../metadata/MetricDefinitionServiceTest.scala | 130 -----
.../metadata/MetricSourceDefinitionTest.scala | 91 ----
.../metrics/adservice/model/RangeTest.scala | 38 --
.../metrics/adservice/model/SeasonTest.scala | 92 ----
ambari-metrics/ambari-metrics-assembly/pom.xml | 148 ------
.../src/main/assembly/anomaly-detection.xml | 60 ---
.../rpm/anomaly-detection/postinstall.sh | 27 -
.../ambari-metrics-grafana/src/main/scripted.js | 118 -----
.../ambari-metrics-timelineservice/pom.xml | 2 +-
.../timeline/HBaseTimelineMetricsService.java | 65 ---
.../metrics/timeline/PhoenixHBaseAccessor.java | 115 ----
.../metrics/timeline/TimelineMetricStore.java | 5 -
.../timeline/query/PhoenixTransactSQL.java | 93 ----
.../webapp/TimelineWebServices.java | 44 --
.../timeline/TestTimelineMetricStore.java | 9 -
ambari-metrics/pom.xml | 3 -
.../AMBARI_METRICS/0.1.0/alerts.json | 70 ---
.../configuration/ams-admanager-config.xml | 115 ----
.../0.1.0/configuration/ams-admanager-env.xml | 107 ----
.../0.1.0/configuration/ams-admanager-log4j.xml | 86 ---
.../configuration/ams-admanager-spark-env.xml | 129 -----
.../AMBARI_METRICS/0.1.0/metainfo.xml | 30 --
.../alert_point_in_time_metric_anomalies.py | 185 -------
.../alerts/alert_trend_metric_anomalies.py | 185 -------
.../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 64 ---
.../0.1.0/package/scripts/ams_admanager.py | 73 ---
.../0.1.0/package/scripts/params.py | 38 --
.../0.1.0/package/scripts/status.py | 2 -
.../0.1.0/package/scripts/status_params.py | 2 -
.../package/templates/admanager_config.yaml.j2 | 44 --
.../2.0.6/AMBARI_METRICS/test_admanager.py | 106 ----
.../python/stacks/2.0.6/configs/default.json | 13 -
.../2.0.6/configs/default_ams_embedded.json | 13 -
139 files changed, 1 insertion(+), 11727 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
index 9aee70f..ded63cf 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
@@ -50,10 +50,6 @@ def should_install_phoenix():
has_phoenix = len(phoenix_hosts) > 0
return phoenix_enabled or has_phoenix
-def should_install_ams_admanager():
- config = Script.get_config()
- return _has_applicable_local_component(config, ["AD_MANAGER"])
-
def should_install_ams_collector():
config = Script.get_config()
return _has_applicable_local_component(config, ["METRICS_COLLECTOR"])
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager
deleted file mode 100644
index 98b7606..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager
+++ /dev/null
@@ -1,227 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific
-
-PIDFILE=/var/run/ambari-metrics-anomaly-detection/ambari-metrics-admanager.pid
-OUTFILE=/var/log/ambari-metrics-anomaly-detection/ambari-metrics-admanager.out
-
-CONF_DIR=/etc/ambari-metrics-anomaly-detection/conf
-DAEMON_NAME=ams_admanager
-SPARK_HOME=/usr/lib/ambari-metrics-anomaly-detection/spark
-
-SPARK_MASTER_PID=/var/run/ambari-metrics-anomaly-detection/spark-ams-org.apache.spark.deploy.master.Master.pid
-
-STOP_TIMEOUT=10
-
-function spark_daemon
-{
- local cmd=$1
- local pid
-
- if [[ "${cmd}" == "start" ]]
- then
-
- ${SPARK_HOME}/sbin/start-master.sh
- sleep 2
- master_pid=$(cat "$SPARK_MASTER_PID")
- if [ -z "`ps ax | grep -w ${master_pid} | grep org.apache.spark.deploy.master.Master`" ]; then
- echo "ERROR: Spark Master start failed. For more details, see outfile in log directory."
- exit -1
- fi
-
- ${SPARK_HOME}/sbin/start-slave.sh spark://${SPARK_MASTER_HOST}:${SPARK_MASTER_PORT}
- elif [[ "${cmd}" == "stop" ]]
- then
- ${SPARK_HOME}/sbin/stop-slave.sh
- ${SPARK_HOME}/sbin/stop-master.sh
- else
- pid=${SPARK_MASTER_PID}
- daemon_status "${pid}"
- fi
-
-}
-
-function write_pidfile
-{
- local pidfile="$1"
- echo $! > "${pidfile}" 2>/dev/null
- if [[ $? -gt 0 ]]; then
- echo "ERROR: Cannot write pid ${pidfile}." | tee -a $STARTUPFILE
- exit 1;
- fi
-}
-
-function java_setup
-{
- # Bail if we did not detect it
- if [[ -z "${JAVA_HOME}" ]]; then
- echo "ERROR: JAVA_HOME is not set and could not be found."
- exit 1
- fi
-
- if [[ ! -d "${JAVA_HOME}" ]]; then
- echo "ERROR: JAVA_HOME ${JAVA_HOME} does not exist."
- exit 1
- fi
-
- JAVA="${JAVA_HOME}/bin/java"
-
- if [[ ! -x "$JAVA" ]]; then
- echo "ERROR: $JAVA is not executable."
- exit 1
- fi
-}
-
-function daemon_status()
-{
- local pidfile="$1"
- shift
-
- local pid
-
- if [[ -f "${pidfile}" ]]; then
- pid=$(cat "${pidfile}")
- if ps -p "${pid}" > /dev/null 2>&1; then
- return 0
- fi
- return 1
- fi
- return 3
-}
-
-function start()
-{
- java_setup
-
-
- if [[ "${AMS_AD_STANDALONE_SPARK_ENABLED}" == "true" || "${AMS_AD_STANDALONE_SPARK_ENABLED}" == "True" ]]
- then
- spark_daemon "start"
- fi
-
- daemon_status "${PIDFILE}"
- if [[ $? == 0 ]]; then
- echo "AMS AD Manager is running as process $(cat "${PIDFILE}"). Exiting" | tee -a $STARTUPFILE
- exit 0
- else
- # stale pid file, so just remove it and continue on
- rm -f "${PIDFILE}" >/dev/null 2>&1
- fi
-
- nohup "${JAVA}" "-Xms$AMS_AD_HEAPSIZE" "-Xmx$AMS_AD_HEAPSIZE" ${AMS_AD_OPTS} "-Dlog4j.configuration=file://$CONF_DIR/log4j.properties" "-jar" "/usr/lib/ambari-metrics-anomaly-detection/ambari-metrics-anomaly-detection-service.jar" "server" "${CONF_DIR}/config.yaml" "$@" > $OUTFILE 2>&1 &
- PID=$!
- write_pidfile "${PIDFILE}"
- sleep 2
-
- echo "Verifying ${DAEMON_NAME} process status..."
- if [ -z "`ps ax -o pid | grep ${PID}`" ]; then
- if [ -s ${OUTFILE} ]; then
- echo "ERROR: ${DAEMON_NAME} start failed. For more details, see ${OUTFILE}:"
- echo "===================="
- tail -n 10 ${OUTFILE}
- echo "===================="
- else
- echo "ERROR: ${DAEMON_NAME} start failed"
- rm -f ${PIDFILE}
- fi
- echo "Anomaly Detection Manager out at: ${OUTFILE}"
- exit -1
- fi
-
- rm -f $STARTUPFILE #Deleting startup file
- echo "Anomaly Detection Manager successfully started."
- }
-
-function stop()
-{
- pidfile=${PIDFILE}
-
- if [[ -f "${pidfile}" ]]; then
- pid=$(cat "$pidfile")
-
- kill "${pid}" >/dev/null 2>&1
- sleep "${STOP_TIMEOUT}"
-
- if kill -0 "${pid}" > /dev/null 2>&1; then
- echo "WARNING: ${DAEMON_NAME} did not stop gracefully after ${STOP_TIMEOUT} seconds: Trying to kill with kill -9"
- kill -9 "${pid}" >/dev/null 2>&1
- fi
-
- if ps -p "${pid}" > /dev/null 2>&1; then
- echo "ERROR: Unable to kill ${pid}"
- else
- rm -f "${pidfile}" >/dev/null 2>&1
- fi
- fi
-
- #Let's try to stop spark always since if the user has flipped the spark mode to 'yarn', the enabled flag becomes obsolete.
- spark_daemon "stop"
-}
-
-# execute ams-admanager-env.sh
-if [[ -f "${CONF_DIR}/ams-admanager-env.sh" ]]; then
- . "${CONF_DIR}/ams-admanager-env.sh"
-else
- echo "ERROR: Cannot execute ${CONF_DIR}/ams-admanager-env.sh." 2>&1
- exit 1
-fi
-
-if [[ -f "${CONF_DIR}/ams-admanager-spark-env.sh" ]]; then
- . "${CONF_DIR}/ams-admanager-spark-env.sh"
-else
- echo "ERROR: Cannot execute ${CONF_DIR}/ams-admanager-spark-env.sh." 2>&1
- exit 1
-fi
-
-# set these env variables only if they were not set by ams-admanager-env.sh
-: ${AMS_AD_LOG_DIR:=/var/log/ambari-metrics-anomaly-detection}
-: ${AMS_AD_STANDALONE_SPARK_ENABLED:=true}
-
-# set pid dir path
-if [[ -n "${AMS_AD_PID_DIR}" ]]; then
- PIDFILE=${AMS_AD_PID_DIR}/ambari-metrics-admanager.pid
- SPARK_MASTER_PID=${AMS_AD_PID_DIR}/spark-${USER}-org.apache.spark.deploy.master.Master-1.pid
-fi
-
-# set out file path
-if [[ -n "${AMS_AD_LOG_DIR}" ]]; then
- OUTFILE=${AMS_AD_LOG_DIR}/ambari-metrics-admanager.out
-fi
-
-#TODO manage 3 hbase daemons for start/stop/status
-case "$1" in
-
- start)
- start
-
- ;;
- stop)
- stop
-
- ;;
- status)
- daemon_status "${PIDFILE}"
- if [[ $? == 0 ]]; then
- echo "AMS AD Manager is running as process $(cat "${PIDFILE}")."
- else
- echo "AMS AD Manager is not running."
- fi
- ;;
- restart)
- stop
- start
- ;;
-
-esac
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/config.yaml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/config.yaml b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/config.yaml
deleted file mode 100644
index 85e4004..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/config.yaml
+++ /dev/null
@@ -1,45 +0,0 @@
-#Licensed under the Apache License, Version 2.0 (the "License");
-#you may not use this file except in compliance with the License.
-#You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
-
-server:
- applicationConnectors:
- - type: http
- port: 9999
- requestLog:
- type: external
-
-logging:
- type: external
-
-metricDefinitionService:
- inputDefinitionDirectory: /etc/ambari-metrics-anomaly-detection/conf/definitionDirectory
-
-metricsCollector:
- hosts: host1,host2
- port: 6188
- protocol: http
- metadataEndpoint: /ws/v1/timeline/metrics/metadata/key
-
-adQueryService:
- anomalyDataTtl: 604800
-
-metricDefinitionDB:
- # force checksum verification of all data that is read from the file system on behalf of a particular read
- verifyChecksums: true
- # raise an error as soon as it detects an internal corruption
- performParanoidChecks: false
- # Path to Level DB directory
- dbDirPath: /tmp/ambari-metrics-anomaly-detection/db
-
-spark:
- mode: standalone
- masterHostPort: localhost:7077
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/log4j.properties
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/log4j.properties b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/log4j.properties
deleted file mode 100644
index 9dba1da..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/log4j.properties
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Define some default values that can be overridden by system properties
-# Root logger option
-log4j.rootLogger=INFO,file
-
-# Direct log messages to a log file
-log4j.appender.file=org.apache.log4j.RollingFileAppender
-log4j.appender.file.File=/var/log/ambari-metrics-anomaly-detection/ambari-metrics-admanager.log
-log4j.appender.file.MaxFileSize=80MB
-log4j.appender.file.MaxBackupIndex=60
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p [%t] %c{1}:%L - %m%n
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml b/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml
deleted file mode 100644
index 50d7ef6..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml
+++ /dev/null
@@ -1,528 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>ambari-metrics</artifactId>
- <groupId>org.apache.ambari</groupId>
- <version>2.0.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>ambari-metrics-anomaly-detection-service</artifactId>
- <version>2.0.0.0-SNAPSHOT</version>
- <name>Ambari Metrics Anomaly Detection Service</name>
- <packaging>jar</packaging>
-
- <properties>
- <scala.version>2.12.3</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
- <jackson.version>2.9.1</jackson.version>
- <dropwizard.version>1.2.0</dropwizard.version>
- <spark.version>2.1.1</spark.version>
- <hadoop.version>2.7.3.2.6.0.3-8</hadoop.version>
- <hbase.version>1.1.2.2.6.0.3-8</hbase.version>
- <phoenix.version>4.7.0.2.6.0.3-8</phoenix.version>
- </properties>
-
- <repositories>
- <repository>
- <id>scala-tools.org</id>
- <name>Scala-Tools Maven2 Repository</name>
- <url>http://scala-tools.org/repo-releases</url>
- </repository>
- </repositories>
-
- <pluginRepositories>
- <pluginRepository>
- <id>scala-tools.org</id>
- <name>Scala-Tools Maven2 Repository</name>
- <url>http://scala-tools.org/repo-releases</url>
- </pluginRepository>
- </pluginRepositories>
-
- <build>
- <finalName>${project.artifactId}-${project.version}</finalName>
- <resources>
- <resource>
- <filtering>true</filtering>
- <directory>src/main/resources</directory>
- <includes>
- <include>**/*.yml</include>
- <include>**/*.xml</include>
- <include>**/*.txt</include>
- </includes>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- </configuration>
- </plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>3.3.1</version>
- <executions>
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>add-source</goal>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <jvmArgs>
- <jvmArg>-Xms512m</jvmArg>
- <jvmArg>-Xmx2048m</jvmArg>
- </jvmArgs>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
- <version>1.0</version>
- </plugin>
- <plugin>
- <groupId>org.scala-tools</groupId>
- <artifactId>maven-scala-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <scalaVersion>${scala.version}</scalaVersion>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>2.5</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.1.0</version>
- <configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <!--<minimizeJar>true</minimizeJar>-->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.phoenix:phoenix-core</artifact>
- <excludes>
- <exclude>org/joda/time/**</exclude>
- <exclude>com/codahale/metrics/**</exclude>
- <exclude>com/google/common/collect/**</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>com/sun/jersey/**</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>
- org.apache.ambari.metrics.adservice.app.AnomalyDetectionApp
- </mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>generate-resources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target name="Download Spark">
- <mkdir dir="${project.build.directory}/embedded"/>
- <get
- src="${spark.tar}"
- dest="${project.build.directory}/embedded/spark.tar.gz"
- usetimestamp="true"
- />
- <untar
- src="${project.build.directory}/embedded/spark.tar.gz"
- dest="${project.build.directory}/embedded"
- compression="gzip"
- />
- <move
- todir="${project.build.directory}/embedded/spark" >
- <fileset dir="${project.build.directory}/embedded/${spark.folder}" includes="**"/>
- </move>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.5</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>1.7.2</version>
- </dependency>
- <dependency>
- <groupId>com.github.lucarosellini.rJava</groupId>
- <artifactId>JRI</artifactId>
- <version>0.9-7</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>0.10.1.0</version>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jdmk</groupId>
- <artifactId>jmxtools</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jmx</groupId>
- <artifactId>jmxri</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.mail</groupId>
- <artifactId>mail</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jms</groupId>
- <artifactId>jmx</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jms</groupId>
- <artifactId>jms</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>0.10.1.0</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>connect-json</artifactId>
- <version>0.10.1.0</version>
- <exclusions>
- <exclusion>
- <artifactId>jackson-databind</artifactId>
- <groupId>com.fasterxml.jackson.core</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-kafka_2.10</artifactId>
- <version>1.6.3</version>
- </dependency>
- <dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-core</artifactId>
- <version>${phoenix.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-core</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-server</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.ambari</groupId>
- <artifactId>ambari-metrics-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.2.5</version>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_2.11</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-mllib_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-server</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-core</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-json</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_2.12</artifactId>
- <version>3.0.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-core</artifactId>
- <version>${dropwizard.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.glassfish.hk2.external</groupId>
- <artifactId>javax.inject</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.glassfish.hk2.external</groupId>
- <artifactId>aopalliance-repackaged</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-access</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-server</artifactId>
- <groupId>org.glassfish.jersey.core</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>1.2.17</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>1.7.21</version>
- </dependency>
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-testing</artifactId>
- <version>${dropwizard.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.glassfish.hk2.external</groupId>
- <artifactId>javax.inject</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- <version>2.9.4</version>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- <version>1.8.1</version>
- </dependency>
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- <version>4.1.0</version>
- </dependency>
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-multibindings</artifactId>
- <version>4.1.0</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_2.12</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-jdk8</artifactId>
- <version>${jackson.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.fusesource.leveldbjni</groupId>
- <artifactId>leveldbjni-all</artifactId>
- <version>1.8</version>
- </dependency>
- <dependency>
- <groupId>org.iq80.leveldb</groupId>
- <artifactId>leveldb</artifactId>
- <version>0.9</version>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.scalaj/scalaj-http -->
- <dependency>
- <groupId>org.scalaj</groupId>
- <artifactId>scalaj-http_2.12</artifactId>
- <version>2.3.0</version>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>18.0</version>
- </dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>3.2.5</version>
- </dependency>
- <dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <version>2.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.8.4</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/assemblies/empty.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/assemblies/empty.xml b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/assemblies/empty.xml
deleted file mode 100644
index 35738b1..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/assemblies/empty.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<assembly>
- <id>empty</id>
- <formats/>
-</assembly>
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/DataSeries.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/DataSeries.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/DataSeries.java
deleted file mode 100644
index 54b402f..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/DataSeries.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.adservice.prototype.common;
-
-import java.util.Arrays;
-
-public class DataSeries {
-
- public String seriesName;
- public double[] ts;
- public double[] values;
-
- public DataSeries(String seriesName, double[] ts, double[] values) {
- this.seriesName = seriesName;
- this.ts = ts;
- this.values = values;
- }
-
- @Override
- public String toString() {
- return seriesName + Arrays.toString(ts) + Arrays.toString(values);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/ResultSet.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/ResultSet.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/ResultSet.java
deleted file mode 100644
index dd3038f..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/ResultSet.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.adservice.prototype.common;
-
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ResultSet {
-
- public List<double[]> resultset = new ArrayList<>();
-
- public ResultSet(List<double[]> resultset) {
- this.resultset = resultset;
- }
-
- public void print() {
- System.out.println("Result : ");
- if (!resultset.isEmpty()) {
- for (int i = 0; i<resultset.get(0).length;i++) {
- for (double[] entity : resultset) {
- System.out.print(entity[i] + " ");
- }
- System.out.println();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/StatisticUtils.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/StatisticUtils.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/StatisticUtils.java
deleted file mode 100644
index 0a22e50..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/StatisticUtils.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.adservice.prototype.common;
-
-
-import java.util.Arrays;
-
-public class StatisticUtils {
-
- public static double mean(double[] values) {
- double sum = 0;
- for (double d : values) {
- sum += d;
- }
- return sum / values.length;
- }
-
- public static double variance(double[] values) {
- double avg = mean(values);
- double variance = 0;
- for (double d : values) {
- variance += Math.pow(d - avg, 2.0);
- }
- return variance;
- }
-
- public static double sdev(double[] values, boolean useBesselsCorrection) {
- double variance = variance(values);
- int n = (useBesselsCorrection) ? values.length - 1 : values.length;
- return Math.sqrt(variance / n);
- }
-
- public static double median(double[] values) {
- double[] clonedValues = Arrays.copyOf(values, values.length);
- Arrays.sort(clonedValues);
- int n = values.length;
-
- if (n % 2 != 0) {
- return clonedValues[(n-1)/2];
- } else {
- return ( clonedValues[(n-1)/2] + clonedValues[n/2] ) / 2;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/AmbariServerInterface.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/AmbariServerInterface.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/AmbariServerInterface.java
deleted file mode 100644
index ac50c54..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/AmbariServerInterface.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.adservice.prototype.core;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Serializable;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-
-public class AmbariServerInterface implements Serializable{
-
- private static final Log LOG = LogFactory.getLog(AmbariServerInterface.class);
-
- private String ambariServerHost;
- private String clusterName;
-
- public AmbariServerInterface(String ambariServerHost, String clusterName) {
- this.ambariServerHost = ambariServerHost;
- this.clusterName = clusterName;
- }
-
- public int getPointInTimeSensitivity() {
-
- String url = constructUri("http", ambariServerHost, "8080", "/api/v1/clusters/" + clusterName + "/alert_definitions?fields=*");
-
- URL obj = null;
- BufferedReader in = null;
-
- try {
- obj = new URL(url);
- HttpURLConnection con = (HttpURLConnection) obj.openConnection();
- con.setRequestMethod("GET");
-
- String encoded = Base64.getEncoder().encodeToString(("admin:admin").getBytes(StandardCharsets.UTF_8));
- con.setRequestProperty("Authorization", "Basic "+encoded);
-
- int responseCode = con.getResponseCode();
- LOG.info("Sending 'GET' request to URL : " + url);
- LOG.info("Response Code : " + responseCode);
-
- in = new BufferedReader(
- new InputStreamReader(con.getInputStream()));
-
- StringBuilder responseJsonSb = new StringBuilder();
- String line;
- while ((line = in.readLine()) != null) {
- responseJsonSb.append(line);
- }
-
-// JSONObject jsonObject = new JSONObject(responseJsonSb.toString());
-// JSONArray array = jsonObject.getJSONArray("items");
-// for(int i = 0 ; i < array.length() ; i++){
-// JSONObject alertDefn = array.getJSONObject(i).getJSONObject("AlertDefinition");
-// if (alertDefn.get("name") != null && alertDefn.get("name").equals("point_in_time_metrics_anomalies")) {
-// JSONObject sourceNode = alertDefn.getJSONObject("source");
-// JSONArray params = sourceNode.getJSONArray("parameters");
-// for(int j = 0 ; j < params.length() ; j++){
-// JSONObject param = params.getJSONObject(j);
-// if (param.get("name").equals("sensitivity")) {
-// return param.getInt("value");
-// }
-// }
-// break;
-// }
-// }
-
- } catch (Exception e) {
- LOG.error(e);
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- LOG.warn(e);
- }
- }
- }
-
- return -1;
- }
-
- private String constructUri(String protocol, String host, String port, String path) {
- StringBuilder sb = new StringBuilder(protocol);
- sb.append("://");
- sb.append(host);
- sb.append(":");
- sb.append(port);
- sb.append(path);
- return sb.toString();
- }
-
-// public static void main(String[] args) {
-// AmbariServerInterface ambariServerInterface = new AmbariServerInterface();
-// ambariServerInterface.getPointInTimeSensitivity("avijayan-ams-1.openstacklocal","c1");
-// }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricKafkaProducer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricKafkaProducer.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricKafkaProducer.java
deleted file mode 100644
index 167fbb3..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricKafkaProducer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.adservice.prototype.core;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.clients.producer.RecordMetadata;
-
-import java.util.Properties;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-
-public class MetricKafkaProducer {
-
- Producer producer;
- private static String topicName = "ambari-metrics-topic";
-
- public MetricKafkaProducer(String kafkaServers) {
- Properties configProperties = new Properties();
- configProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServers); //"avijayan-ams-2.openstacklocal:6667"
- configProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.ByteArraySerializer");
- configProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.connect.json.JsonSerializer");
- producer = new KafkaProducer(configProperties);
- }
-
- public void sendMetrics(TimelineMetrics timelineMetrics) throws InterruptedException, ExecutionException {
-
- ObjectMapper objectMapper = new ObjectMapper();
- JsonNode jsonNode = objectMapper.valueToTree(timelineMetrics);
- ProducerRecord<String, JsonNode> rec = new ProducerRecord<String, JsonNode>(topicName,jsonNode);
- Future<RecordMetadata> kafkaFuture = producer.send(rec);
-
- System.out.println(kafkaFuture.isDone());
- System.out.println(kafkaFuture.get().topic());
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricSparkConsumer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricSparkConsumer.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricSparkConsumer.java
deleted file mode 100644
index addeda7..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricSparkConsumer.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.adservice.prototype.core;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
-import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaTechnique;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.broadcast.Broadcast;
-import org.apache.spark.streaming.Duration;
-import org.apache.spark.streaming.api.java.JavaDStream;
-import org.apache.spark.streaming.api.java.JavaPairDStream;
-import org.apache.spark.streaming.api.java.JavaPairReceiverInputDStream;
-import org.apache.spark.streaming.api.java.JavaStreamingContext;
-import org.apache.spark.streaming.kafka.KafkaUtils;
-import scala.Tuple2;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Properties;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class MetricSparkConsumer {
-
- private static final Log LOG = LogFactory.getLog(MetricSparkConsumer.class);
- private static String groupId = "ambari-metrics-group";
- private static String topicName = "ambari-metrics-topic";
- private static int numThreads = 1;
- private static long pitStartTime = System.currentTimeMillis();
- private static long ksStartTime = pitStartTime;
- private static long hdevStartTime = ksStartTime;
- private static Set<Pattern> includeMetricPatterns = new HashSet<>();
- private static Set<String> includedHosts = new HashSet<>();
- private static Set<TrendMetric> trendMetrics = new HashSet<>();
-
- public MetricSparkConsumer() {
- }
-
- public static Properties readProperties(String propertiesFile) {
- try {
- Properties properties = new Properties();
- InputStream inputStream = ClassLoader.getSystemResourceAsStream(propertiesFile);
- if (inputStream == null) {
- inputStream = new FileInputStream(propertiesFile);
- }
- properties.load(inputStream);
- return properties;
- } catch (IOException ioEx) {
- LOG.error("Error reading properties file for jmeter");
- return null;
- }
- }
-
- public static void main(String[] args) throws InterruptedException {
-
- if (args.length < 1) {
- System.err.println("Usage: MetricSparkConsumer <input-config-file>");
- System.exit(1);
- }
-
- Properties properties = readProperties(args[0]);
-
- List<String> appIds = Arrays.asList(properties.getProperty("appIds").split(","));
-
- String collectorHost = properties.getProperty("collectorHost");
- String collectorPort = properties.getProperty("collectorPort");
- String collectorProtocol = properties.getProperty("collectorProtocol");
-
- String zkQuorum = properties.getProperty("zkQuorum");
-
- double emaW = Double.parseDouble(properties.getProperty("emaW"));
- double emaN = Double.parseDouble(properties.getProperty("emaN"));
- int emaThreshold = Integer.parseInt(properties.getProperty("emaThreshold"));
- double tukeysN = Double.parseDouble(properties.getProperty("tukeysN"));
-
- long pitTestInterval = Long.parseLong(properties.getProperty("pointInTimeTestInterval"));
- long pitTrainInterval = Long.parseLong(properties.getProperty("pointInTimeTrainInterval"));
-
- long ksTestInterval = Long.parseLong(properties.getProperty("ksTestInterval"));
- long ksTrainInterval = Long.parseLong(properties.getProperty("ksTrainInterval"));
- int hsdevNhp = Integer.parseInt(properties.getProperty("hsdevNhp"));
- long hsdevInterval = Long.parseLong(properties.getProperty("hsdevInterval"));
-
- String ambariServerHost = properties.getProperty("ambariServerHost");
- String clusterName = properties.getProperty("clusterName");
-
- String includeMetricPatternStrings = properties.getProperty("includeMetricPatterns");
- if (includeMetricPatternStrings != null && !includeMetricPatternStrings.isEmpty()) {
- String[] patterns = includeMetricPatternStrings.split(",");
- for (String p : patterns) {
- LOG.info("Included Pattern : " + p);
- includeMetricPatterns.add(Pattern.compile(p));
- }
- }
-
- String includedHostList = properties.getProperty("hosts");
- if (includedHostList != null && !includedHostList.isEmpty()) {
- String[] hosts = includedHostList.split(",");
- includedHosts.addAll(Arrays.asList(hosts));
- }
-
- MetricsCollectorInterface metricsCollectorInterface = new MetricsCollectorInterface(collectorHost, collectorProtocol, collectorPort);
-
- SparkConf sparkConf = new SparkConf().setAppName("AmbariMetricsAnomalyDetector");
-
- JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, new Duration(10000));
-
- EmaTechnique emaTechnique = new EmaTechnique(emaW, emaN, emaThreshold);
- PointInTimeADSystem pointInTimeADSystem = new PointInTimeADSystem(metricsCollectorInterface,
- tukeysN,
- pitTestInterval,
- pitTrainInterval,
- ambariServerHost,
- clusterName);
-
- TrendADSystem trendADSystem = new TrendADSystem(metricsCollectorInterface,
- ksTestInterval,
- ksTrainInterval,
- hsdevNhp);
-
- Broadcast<EmaTechnique> emaTechniqueBroadcast = jssc.sparkContext().broadcast(emaTechnique);
- Broadcast<PointInTimeADSystem> pointInTimeADSystemBroadcast = jssc.sparkContext().broadcast(pointInTimeADSystem);
- Broadcast<TrendADSystem> trendADSystemBroadcast = jssc.sparkContext().broadcast(trendADSystem);
- Broadcast<MetricsCollectorInterface> metricsCollectorInterfaceBroadcast = jssc.sparkContext().broadcast(metricsCollectorInterface);
- Broadcast<Set<Pattern>> includePatternBroadcast = jssc.sparkContext().broadcast(includeMetricPatterns);
- Broadcast<Set<String>> includedHostBroadcast = jssc.sparkContext().broadcast(includedHosts);
-
- JavaPairReceiverInputDStream<String, String> messages =
- KafkaUtils.createStream(jssc, zkQuorum, groupId, Collections.singletonMap(topicName, numThreads));
-
- //Convert JSON string to TimelineMetrics.
- JavaDStream<TimelineMetrics> timelineMetricsStream = messages.map(new Function<Tuple2<String, String>, TimelineMetrics>() {
- @Override
- public TimelineMetrics call(Tuple2<String, String> message) throws Exception {
- ObjectMapper mapper = new ObjectMapper();
- TimelineMetrics metrics = mapper.readValue(message._2, TimelineMetrics.class);
- return metrics;
- }
- });
-
- timelineMetricsStream.print();
-
- //Group TimelineMetric by AppId.
- JavaPairDStream<String, TimelineMetrics> appMetricStream = timelineMetricsStream.mapToPair(
- timelineMetrics -> timelineMetrics.getMetrics().isEmpty() ? new Tuple2<>("TEST", new TimelineMetrics()) : new Tuple2<String, TimelineMetrics>(timelineMetrics.getMetrics().get(0).getAppId(), timelineMetrics)
- );
-
- appMetricStream.print();
-
- //Filter AppIds that are not needed.
- JavaPairDStream<String, TimelineMetrics> filteredAppMetricStream = appMetricStream.filter(new Function<Tuple2<String, TimelineMetrics>, Boolean>() {
- @Override
- public Boolean call(Tuple2<String, TimelineMetrics> appMetricTuple) throws Exception {
- return appIds.contains(appMetricTuple._1);
- }
- });
-
- filteredAppMetricStream.print();
-
- filteredAppMetricStream.foreachRDD(rdd -> {
- rdd.foreach(
- tuple2 -> {
- long currentTime = System.currentTimeMillis();
- EmaTechnique ema = emaTechniqueBroadcast.getValue();
- if (currentTime > pitStartTime + pitTestInterval) {
- LOG.info("Running Tukeys....");
- pointInTimeADSystemBroadcast.getValue().runTukeysAndRefineEma(ema, currentTime);
- pitStartTime = pitStartTime + pitTestInterval;
- }
-
- if (currentTime > ksStartTime + ksTestInterval) {
- LOG.info("Running KS Test....");
- trendADSystemBroadcast.getValue().runKSTest(currentTime, trendMetrics);
- ksStartTime = ksStartTime + ksTestInterval;
- }
-
- if (currentTime > hdevStartTime + hsdevInterval) {
- LOG.info("Running HSdev Test....");
- trendADSystemBroadcast.getValue().runHsdevMethod();
- hdevStartTime = hdevStartTime + hsdevInterval;
- }
-
- TimelineMetrics metrics = tuple2._2();
- for (TimelineMetric timelineMetric : metrics.getMetrics()) {
-
- boolean includeHost = includedHostBroadcast.getValue().contains(timelineMetric.getHostName());
- boolean includeMetric = false;
- if (includeHost) {
- if (includePatternBroadcast.getValue().isEmpty()) {
- includeMetric = true;
- }
- for (Pattern p : includePatternBroadcast.getValue()) {
- Matcher m = p.matcher(timelineMetric.getMetricName());
- if (m.find()) {
- includeMetric = true;
- }
- }
- }
-
- if (includeMetric) {
- trendMetrics.add(new TrendMetric(timelineMetric.getMetricName(), timelineMetric.getAppId(),
- timelineMetric.getHostName()));
- List<MetricAnomaly> anomalies = ema.test(timelineMetric);
- metricsCollectorInterfaceBroadcast.getValue().publish(anomalies);
- }
- }
- });
- });
-
- jssc.start();
- jssc.awaitTermination();
- }
-}
-
-
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricsCollectorInterface.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricsCollectorInterface.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricsCollectorInterface.java
deleted file mode 100644
index da3999a..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricsCollectorInterface.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.adservice.prototype.core;
-
-import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.codehaus.jackson.map.AnnotationIntrospector;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.ObjectReader;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.Serializable;
-import java.net.HttpURLConnection;
-import java.net.InetAddress;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.TreeMap;
-
-public class MetricsCollectorInterface implements Serializable {
-
- private static String hostName = null;
- private String instanceId = null;
- public final static String serviceName = "anomaly-engine";
- private String collectorHost;
- private String protocol;
- private String port;
- private static final String WS_V1_TIMELINE_METRICS = "/ws/v1/timeline/metrics";
- private static final Log LOG = LogFactory.getLog(MetricsCollectorInterface.class);
- private static ObjectMapper mapper;
- private final static ObjectReader timelineObjectReader;
-
- static {
- mapper = new ObjectMapper();
- AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
- mapper.setAnnotationIntrospector(introspector);
- mapper.getSerializationConfig()
- .withSerializationInclusion(JsonSerialize.Inclusion.NON_NULL);
- timelineObjectReader = mapper.reader(TimelineMetrics.class);
- }
-
- public MetricsCollectorInterface(String collectorHost, String protocol, String port) {
- this.collectorHost = collectorHost;
- this.protocol = protocol;
- this.port = port;
- this.hostName = getDefaultLocalHostName();
- }
-
- public static String getDefaultLocalHostName() {
-
- if (hostName != null) {
- return hostName;
- }
-
- try {
- return InetAddress.getLocalHost().getCanonicalHostName();
- } catch (UnknownHostException e) {
- LOG.info("Error getting host address");
- }
- return null;
- }
-
- public void publish(List<MetricAnomaly> metricAnomalies) {
- if (CollectionUtils.isNotEmpty(metricAnomalies)) {
- LOG.info("Sending metric anomalies of size : " + metricAnomalies.size());
- List<TimelineMetric> metricList = getTimelineMetricList(metricAnomalies);
- if (!metricList.isEmpty()) {
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- timelineMetrics.setMetrics(metricList);
- emitMetrics(timelineMetrics);
- }
- } else {
- LOG.debug("No anomalies to send.");
- }
- }
-
- private List<TimelineMetric> getTimelineMetricList(List<MetricAnomaly> metricAnomalies) {
- List<TimelineMetric> metrics = new ArrayList<>();
-
- if (metricAnomalies.isEmpty()) {
- return metrics;
- }
-
- for (MetricAnomaly anomaly : metricAnomalies) {
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName(anomaly.getMetricKey());
- timelineMetric.setAppId(serviceName + "-" + anomaly.getMethodType());
- timelineMetric.setInstanceId(null);
- timelineMetric.setHostName(getDefaultLocalHostName());
- timelineMetric.setStartTime(anomaly.getTimestamp());
- HashMap<String, String> metadata = new HashMap<>();
- metadata.put("method", anomaly.getMethodType());
- metadata.put("anomaly-score", String.valueOf(anomaly.getAnomalyScore()));
- timelineMetric.setMetadata(metadata);
- TreeMap<Long,Double> metricValues = new TreeMap<>();
- metricValues.put(anomaly.getTimestamp(), anomaly.getMetricValue());
- timelineMetric.setMetricValues(metricValues);
-
- metrics.add(timelineMetric);
- }
- return metrics;
- }
-
- public boolean emitMetrics(TimelineMetrics metrics) {
- String connectUrl = constructTimelineMetricUri();
- String jsonData = null;
- LOG.debug("EmitMetrics connectUrl = " + connectUrl);
- try {
- jsonData = mapper.writeValueAsString(metrics);
- LOG.info(jsonData);
- } catch (IOException e) {
- LOG.error("Unable to parse metrics", e);
- }
- if (jsonData != null) {
- return emitMetricsJson(connectUrl, jsonData);
- }
- return false;
- }
-
- private HttpURLConnection getConnection(String spec) throws IOException {
- return (HttpURLConnection) new URL(spec).openConnection();
- }
-
- private boolean emitMetricsJson(String connectUrl, String jsonData) {
- int timeout = 10000;
- HttpURLConnection connection = null;
- try {
- if (connectUrl == null) {
- throw new IOException("Unknown URL. Unable to connect to metrics collector.");
- }
- connection = getConnection(connectUrl);
-
- connection.setRequestMethod("POST");
- connection.setRequestProperty("Content-Type", "application/json");
- connection.setRequestProperty("Connection", "Keep-Alive");
- connection.setConnectTimeout(timeout);
- connection.setReadTimeout(timeout);
- connection.setDoOutput(true);
-
- if (jsonData != null) {
- try (OutputStream os = connection.getOutputStream()) {
- os.write(jsonData.getBytes("UTF-8"));
- }
- }
-
- int statusCode = connection.getResponseCode();
-
- if (statusCode != 200) {
- LOG.info("Unable to POST metrics to collector, " + connectUrl + ", " +
- "statusCode = " + statusCode);
- } else {
- LOG.info("Metrics posted to Collector " + connectUrl);
- }
- return true;
- } catch (IOException ioe) {
- LOG.error(ioe.getMessage());
- }
- return false;
- }
-
- private String constructTimelineMetricUri() {
- StringBuilder sb = new StringBuilder(protocol);
- sb.append("://");
- sb.append(collectorHost);
- sb.append(":");
- sb.append(port);
- sb.append(WS_V1_TIMELINE_METRICS);
- return sb.toString();
- }
-
- public TimelineMetrics fetchMetrics(String metricName,
- String appId,
- String hostname,
- long startime,
- long endtime) {
-
- String url = constructTimelineMetricUri() + "?metricNames=" + metricName + "&appId=" + appId +
- "&hostname=" + hostname + "&startTime=" + startime + "&endTime=" + endtime;
- LOG.debug("Fetch metrics URL : " + url);
-
- URL obj = null;
- BufferedReader in = null;
- TimelineMetrics timelineMetrics = new TimelineMetrics();
-
- try {
- obj = new URL(url);
- HttpURLConnection con = (HttpURLConnection) obj.openConnection();
- con.setRequestMethod("GET");
- int responseCode = con.getResponseCode();
- LOG.debug("Sending 'GET' request to URL : " + url);
- LOG.debug("Response Code : " + responseCode);
-
- in = new BufferedReader(
- new InputStreamReader(con.getInputStream()));
- timelineMetrics = timelineObjectReader.readValue(in);
- } catch (Exception e) {
- LOG.error(e);
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- LOG.warn(e);
- }
- }
- }
-
- LOG.info("Fetched " + timelineMetrics.getMetrics().size() + " metrics.");
- return timelineMetrics;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/25c18121/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/PointInTimeADSystem.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/PointInTimeADSystem.java b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/PointInTimeADSystem.java
deleted file mode 100644
index f379605..0000000
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/PointInTimeADSystem.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.metrics.adservice.prototype.core;
-
-import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
-import org.apache.ambari.metrics.adservice.prototype.common.ResultSet;
-import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaModel;
-import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaTechnique;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-public class PointInTimeADSystem implements Serializable {
-
- //private EmaTechnique emaTechnique;
- private MetricsCollectorInterface metricsCollectorInterface;
- private Map<String, Double> tukeysNMap;
- private double defaultTukeysN = 3;
-
- private long testIntervalMillis = 5*60*1000; //10mins
- private long trainIntervalMillis = 15*60*1000; //1hour
-
- private static final Log LOG = LogFactory.getLog(PointInTimeADSystem.class);
-
- private AmbariServerInterface ambariServerInterface;
- private int sensitivity = 50;
- private int minSensitivity = 0;
- private int maxSensitivity = 100;
-
- public PointInTimeADSystem(MetricsCollectorInterface metricsCollectorInterface, double defaultTukeysN,
- long testIntervalMillis, long trainIntervalMillis, String ambariServerHost, String clusterName) {
- this.metricsCollectorInterface = metricsCollectorInterface;
- this.defaultTukeysN = defaultTukeysN;
- this.tukeysNMap = new HashMap<>();
- this.testIntervalMillis = testIntervalMillis;
- this.trainIntervalMillis = trainIntervalMillis;
- this.ambariServerInterface = new AmbariServerInterface(ambariServerHost, clusterName);
- LOG.info("Starting PointInTimeADSystem...");
- }
-
- public void runTukeysAndRefineEma(EmaTechnique emaTechnique, long startTime) {
- LOG.info("Running Tukeys for test data interval [" + new Date(startTime - testIntervalMillis) + " : " + new Date(startTime) + "], with train data period [" + new Date(startTime - testIntervalMillis - trainIntervalMillis) + " : " + new Date(startTime - testIntervalMillis) + "]");
-
- int requiredSensivity = ambariServerInterface.getPointInTimeSensitivity();
- if (requiredSensivity == -1 || requiredSensivity == sensitivity) {
- LOG.info("No change in sensitivity needed.");
- } else {
- LOG.info("Current tukey's N value = " + defaultTukeysN);
- if (requiredSensivity > sensitivity) {
- int targetSensitivity = Math.min(maxSensitivity, requiredSensivity);
- while (sensitivity < targetSensitivity) {
- defaultTukeysN = defaultTukeysN + defaultTukeysN * 0.05;
- sensitivity++;
- }
- } else {
- int targetSensitivity = Math.max(minSensitivity, requiredSensivity);
- while (sensitivity > targetSensitivity) {
- defaultTukeysN = defaultTukeysN - defaultTukeysN * 0.05;
- sensitivity--;
- }
- }
- LOG.info("New tukey's N value = " + defaultTukeysN);
- }
-
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- for (String metricKey : emaTechnique.getTrackedEmas().keySet()) {
- LOG.info("EMA key = " + metricKey);
- EmaModel emaModel = emaTechnique.getTrackedEmas().get(metricKey);
- String metricName = emaModel.getMetricName();
- String appId = emaModel.getAppId();
- String hostname = emaModel.getHostname();
-
- TimelineMetrics tukeysData = metricsCollectorInterface.fetchMetrics(metricName, appId, hostname, startTime - (testIntervalMillis + trainIntervalMillis),
- startTime);
-
- if (tukeysData.getMetrics().isEmpty()) {
- LOG.info("No metrics fetched for Tukeys, metricKey = " + metricKey);
- continue;
- }
-
- List<Double> trainTsList = new ArrayList<>();
- List<Double> trainDataList = new ArrayList<>();
- List<Double> testTsList = new ArrayList<>();
- List<Double> testDataList = new ArrayList<>();
-
- for (TimelineMetric metric : tukeysData.getMetrics()) {
- for (Long timestamp : metric.getMetricValues().keySet()) {
- if (timestamp <= (startTime - testIntervalMillis)) {
- trainDataList.add(metric.getMetricValues().get(timestamp));
- trainTsList.add((double)timestamp);
- } else {
- testDataList.add(metric.getMetricValues().get(timestamp));
- testTsList.add((double)timestamp);
- }
- }
- }
-
- if (trainDataList.isEmpty() || testDataList.isEmpty() || trainDataList.size() < testDataList.size()) {
- LOG.info("Not enough train/test data to perform analysis.");
- continue;
- }
-
- String tukeysTrainSeries = "tukeysTrainSeries";
- double[] trainTs = new double[trainTsList.size()];
- double[] trainData = new double[trainTsList.size()];
- for (int i = 0; i < trainTs.length; i++) {
- trainTs[i] = trainTsList.get(i);
- trainData[i] = trainDataList.get(i);
- }
-
- String tukeysTestSeries = "tukeysTestSeries";
- double[] testTs = new double[testTsList.size()];
- double[] testData = new double[testTsList.size()];
- for (int i = 0; i < testTs.length; i++) {
- testTs[i] = testTsList.get(i);
- testData[i] = testDataList.get(i);
- }
-
- LOG.info("Train Size = " + trainTs.length + ", Test Size = " + testTs.length);
-
- DataSeries tukeysTrainData = new DataSeries(tukeysTrainSeries, trainTs, trainData);
- DataSeries tukeysTestData = new DataSeries(tukeysTestSeries, testTs, testData);
-
- if (!tukeysNMap.containsKey(metricKey)) {
- tukeysNMap.put(metricKey, defaultTukeysN);
- }
-
- Map<String, String> configs = new HashMap<>();
- configs.put("tukeys.n", String.valueOf(tukeysNMap.get(metricKey)));
-
- ResultSet rs = RFunctionInvoker.tukeys(tukeysTrainData, tukeysTestData, configs);
-
- List<TimelineMetric> tukeysMetrics = getAsTimelineMetric(rs, metricName, appId, hostname);
- LOG.info("Tukeys anomalies size : " + tukeysMetrics.size());
- TreeMap<Long, Double> tukeysMetricValues = new TreeMap<>();
-
- for (TimelineMetric tukeysMetric : tukeysMetrics) {
- tukeysMetricValues.putAll(tukeysMetric.getMetricValues());
- timelineMetrics.addOrMergeTimelineMetric(tukeysMetric);
- }
-
- TimelineMetrics emaData = metricsCollectorInterface.fetchMetrics(metricKey, MetricsCollectorInterface.serviceName+"-ema", MetricsCollectorInterface.getDefaultLocalHostName(), startTime - testIntervalMillis, startTime);
- TreeMap<Long, Double> emaMetricValues = new TreeMap();
- if (!emaData.getMetrics().isEmpty()) {
- emaMetricValues = emaData.getMetrics().get(0).getMetricValues();
- }
-
- LOG.info("Ema anomalies size : " + emaMetricValues.size());
- int tp = 0;
- int tn = 0;
- int fp = 0;
- int fn = 0;
-
- for (double ts : testTs) {
- long timestamp = (long) ts;
- if (tukeysMetricValues.containsKey(timestamp)) {
- if (emaMetricValues.containsKey(timestamp)) {
- tp++;
- } else {
- fn++;
- }
- } else {
- if (emaMetricValues.containsKey(timestamp)) {
- fp++;
- } else {
- tn++;
- }
- }
- }
-
- double recall = (double) tp / (double) (tp + fn);
- double precision = (double) tp / (double) (tp + fp);
- LOG.info("----------------------------");
- LOG.info("Precision Recall values for " + metricKey);
- LOG.info("tp=" + tp + ", fp=" + fp + ", tn=" + tn + ", fn=" + fn);
- LOG.info("----------------------------");
-
- if (recall < 0.5) {
- LOG.info("Increasing EMA sensitivity by 10%");
- emaModel.updateModel(true, 5);
- } else if (precision < 0.5) {
- LOG.info("Decreasing EMA sensitivity by 10%");
- emaModel.updateModel(false, 5);
- }
-
- }
-
- if (emaTechnique.getTrackedEmas().isEmpty()){
- LOG.info("No EMA Technique keys tracked!!!!");
- }
-
- if (!timelineMetrics.getMetrics().isEmpty()) {
- metricsCollectorInterface.emitMetrics(timelineMetrics);
- }
- }
-
- private static List<TimelineMetric> getAsTimelineMetric(ResultSet result, String metricName, String appId, String hostname) {
-
- List<TimelineMetric> timelineMetrics = new ArrayList<>();
-
- if (result == null) {
- LOG.info("ResultSet from R call is null!!");
- return null;
- }
-
- if (result.resultset.size() > 0) {
- double[] ts = result.resultset.get(0);
- double[] metrics = result.resultset.get(1);
- double[] anomalyScore = result.resultset.get(2);
- for (int i = 0; i < ts.length; i++) {
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName(metricName + ":" + appId + ":" + hostname);
- timelineMetric.setHostName(MetricsCollectorInterface.getDefaultLocalHostName());
- timelineMetric.setAppId(MetricsCollectorInterface.serviceName + "-tukeys");
- timelineMetric.setInstanceId(null);
- timelineMetric.setStartTime((long) ts[i]);
- TreeMap<Long, Double> metricValues = new TreeMap<>();
- metricValues.put((long) ts[i], metrics[i]);
-
- HashMap<String, String> metadata = new HashMap<>();
- metadata.put("method", "tukeys");
- if (String.valueOf(anomalyScore[i]).equals("infinity")) {
- LOG.info("Got anomalyScore = infinity for " + metricName + ":" + appId + ":" + hostname);
- } else {
- metadata.put("anomaly-score", String.valueOf(anomalyScore[i]));
- }
- timelineMetric.setMetadata(metadata);
-
- timelineMetric.setMetricValues(metricValues);
- timelineMetrics.add(timelineMetric);
- }
- }
-
- return timelineMetrics;
- }
-}