You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2018/04/01 19:14:26 UTC
[ambari] 30/39: AMBARI-22567 : Integrate Spark lifecycle management
into AMS AD Manager. (avijayan)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit d46abc7ac19c4316463214022b28c95d63058764
Author: Aravindan Vijayan <av...@hortonworks.com>
AuthorDate: Thu Nov 30 16:07:56 2017 -0800
AMBARI-22567 : Integrate Spark lifecycle management into AMS AD Manager. (avijayan)
---
...trics-admanager.sh => ambari-metrics-admanager} | 75 ++++++++---
.../resources/config.yml => conf/unix/config.yaml} | 9 +-
.../pom.xml | 44 ++++--
.../src/main/assemblies/empty.xml | 21 +++
.../adservice/app/AnomalyDetectionAppConfig.scala | 10 ++
.../MetricDefinitionServiceConfiguration.scala | 3 -
.../configuration/SparkConfiguration.scala | 39 ++++++
.../adservice/db/PhoenixAnomalyStoreAccessor.scala | 5 +-
.../PointInTimeAnomalyInstance.scala | 4 +-
.../adservice/{common => model}/Range.scala | 2 +-
.../adservice/{common => model}/Season.scala | 4 +-
.../adservice/{common => model}/SeasonType.scala | 2 +-
.../adservice/{common => model}/TimeRange.scala | 2 +-
.../trend => model}/TrendAnomalyInstance.scala | 4 +-
.../config.yml => test/resources/config.yaml} | 17 +--
.../app/AnomalyDetectionAppConfigTest.scala | 19 +--
.../adservice/app/DefaultADResourceSpecTest.scala | 2 +-
.../adservice/{common => model}/RangeTest.scala | 7 +-
.../adservice/{common => model}/SeasonTest.scala | 19 +--
ambari-metrics/ambari-metrics-assembly/pom.xml | 148 +++++++++++++++++++++
.../src/main/assembly/anomaly-detection.xml | 60 +++++++++
.../package/rpm/anomaly-detection/postinstall.sh | 27 ++++
ambari-metrics/pom.xml | 4 +-
.../0.1.0/configuration/ams-admanager-config.xml | 4 +
.../0.1.0/configuration/ams-admanager-env.xml | 6 +-
.../0.1.0/configuration/ams-admanager-log4j.xml | 2 +-
.../configuration/ams-admanager-spark-env.xml | 129 ++++++++++++++++++
.../AMBARI_METRICS/0.1.0/metainfo.xml | 1 +
.../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 14 +-
.../0.1.0/package/scripts/ams_admanager.py | 4 +-
.../AMBARI_METRICS/0.1.0/package/scripts/params.py | 22 ++-
.../0.1.0/package/scripts/status_params.py | 2 +-
32 files changed, 613 insertions(+), 98 deletions(-)
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager.sh b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager
similarity index 70%
rename from ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager.sh
rename to ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager
index f1a1ae3..98b7606 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager.sh
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/ambari-metrics-admanager
@@ -14,13 +14,44 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific
-PIDFILE=/var/run//var/run/ambari-metrics-anomaly-detection/ambari-metrics-admanager.pid
+PIDFILE=/var/run/ambari-metrics-anomaly-detection/ambari-metrics-admanager.pid
OUTFILE=/var/log/ambari-metrics-anomaly-detection/ambari-metrics-admanager.out
CONF_DIR=/etc/ambari-metrics-anomaly-detection/conf
DAEMON_NAME=ams_admanager
+SPARK_HOME=/usr/lib/ambari-metrics-anomaly-detection/spark
-STOP_TIMEOUT=5
+SPARK_MASTER_PID=/var/run/ambari-metrics-anomaly-detection/spark-ams-org.apache.spark.deploy.master.Master.pid
+
+STOP_TIMEOUT=10
+
+function spark_daemon
+{
+ local cmd=$1
+ local pid
+
+ if [[ "${cmd}" == "start" ]]
+ then
+
+ ${SPARK_HOME}/sbin/start-master.sh
+ sleep 2
+ master_pid=$(cat "$SPARK_MASTER_PID")
+ if [ -z "`ps ax | grep -w ${master_pid} | grep org.apache.spark.deploy.master.Master`" ]; then
+ echo "ERROR: Spark Master start failed. For more details, see outfile in log directory."
+ exit -1
+ fi
+
+ ${SPARK_HOME}/sbin/start-slave.sh spark://${SPARK_MASTER_HOST}:${SPARK_MASTER_PORT}
+ elif [[ "${cmd}" == "stop" ]]
+ then
+ ${SPARK_HOME}/sbin/stop-slave.sh
+ ${SPARK_HOME}/sbin/stop-master.sh
+ else
+ pid=${SPARK_MASTER_PID}
+ daemon_status "${pid}"
+ fi
+
+}
function write_pidfile
{
@@ -55,22 +86,6 @@ function java_setup
function daemon_status()
{
- #
- # LSB 4.1.0 compatible status command (1)
- #
- # 0 = program is running
- # 1 = dead, but still a pid (2)
- # 2 = (not used by us)
- # 3 = not running
- #
- # 1 - this is not an endorsement of the LSB
- #
- # 2 - technically, the specification says /var/run/pid, so
- # we should never return this value, but we're giving
- # them the benefit of a doubt and returning 1 even if
- # our pid is not in in /var/run .
- #
-
local pidfile="$1"
shift
@@ -90,6 +105,12 @@ function start()
{
java_setup
+
+ if [[ "${AMS_AD_STANDALONE_SPARK_ENABLED}" == "true" || "${AMS_AD_STANDALONE_SPARK_ENABLED}" == "True" ]]
+ then
+ spark_daemon "start"
+ fi
+
daemon_status "${PIDFILE}"
if [[ $? == 0 ]]; then
echo "AMS AD Manager is running as process $(cat "${PIDFILE}"). Exiting" | tee -a $STARTUPFILE
@@ -144,9 +165,12 @@ function stop()
rm -f "${pidfile}" >/dev/null 2>&1
fi
fi
+
+ #Let's try to stop spark always since if the user has flipped the spark mode to 'yarn', the enabled flag becomes obsolete.
+ spark_daemon "stop"
}
-# execute ams-env.sh
+# execute ams-admanager-env.sh
if [[ -f "${CONF_DIR}/ams-admanager-env.sh" ]]; then
. "${CONF_DIR}/ams-admanager-env.sh"
else
@@ -154,12 +178,21 @@ else
exit 1
fi
-# set these env variables only if they were not set by ams-env.sh
+if [[ -f "${CONF_DIR}/ams-admanager-spark-env.sh" ]]; then
+ . "${CONF_DIR}/ams-admanager-spark-env.sh"
+else
+ echo "ERROR: Cannot execute ${CONF_DIR}/ams-admanager-spark-env.sh." 2>&1
+ exit 1
+fi
+
+# set these env variables only if they were not set by ams-admanager-env.sh
: ${AMS_AD_LOG_DIR:=/var/log/ambari-metrics-anomaly-detection}
+: ${AMS_AD_STANDALONE_SPARK_ENABLED:=true}
# set pid dir path
if [[ -n "${AMS_AD_PID_DIR}" ]]; then
- PIDFILE=${AMS_AD_PID_DIR}/admanager.pid
+ PIDFILE=${AMS_AD_PID_DIR}/ambari-metrics-admanager.pid
+ SPARK_MASTER_PID=${AMS_AD_PID_DIR}/spark-${USER}-org.apache.spark.deploy.master.Master-1.pid
fi
# set out file path
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/config.yaml
similarity index 91%
copy from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml
copy to ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/config.yaml
index 7de06b4..85e4004 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/conf/unix/config.yaml
@@ -38,9 +38,8 @@ metricDefinitionDB:
# raise an error as soon as it detects an internal corruption
performParanoidChecks: false
# Path to Level DB directory
- dbDirPath: /var/lib/ambari-metrics-anomaly-detection/
+ dbDirPath: /tmp/ambari-metrics-anomaly-detection/db
-#subsystemService:
-# spark:
-# pointInTime:
-# trend:
\ No newline at end of file
+spark:
+ mode: standalone
+ masterHostPort: localhost:7077
\ No newline at end of file
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml b/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml
index c6927dd..50d7ef6 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/pom.xml
@@ -36,7 +36,7 @@
<scala.binary.version>2.11</scala.binary.version>
<jackson.version>2.9.1</jackson.version>
<dropwizard.version>1.2.0</dropwizard.version>
- <spark.version>2.2.0</spark.version>
+ <spark.version>2.1.1</spark.version>
<hadoop.version>2.7.3.2.6.0.3-8</hadoop.version>
<hbase.version>1.1.2.2.6.0.3-8</hbase.version>
<phoenix.version>4.7.0.2.6.0.3-8</phoenix.version>
@@ -59,7 +59,7 @@
</pluginRepositories>
<build>
- <finalName>${project.artifactId}</finalName>
+ <finalName>${project.artifactId}-${project.version}</finalName>
<resources>
<resource>
<filtering>true</filtering>
@@ -157,14 +157,6 @@
</excludes>
</filter>
<filter>
- <artifact>org.apache.phoenix:phoenix-core</artifact>
- <excludes>
- <exclude>org/joda/time/**</exclude>
- <exclude>com/codahale/metrics/**</exclude>
- <exclude>com/google/common/collect/**</exclude>
- </excludes>
- </filter>
- <filter>
<artifact>*:*</artifact>
<excludes>
<exclude>com/sun/jersey/**</exclude>
@@ -191,6 +183,38 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target name="Download Spark">
+ <mkdir dir="${project.build.directory}/embedded"/>
+ <get
+ src="${spark.tar}"
+ dest="${project.build.directory}/embedded/spark.tar.gz"
+ usetimestamp="true"
+ />
+ <untar
+ src="${project.build.directory}/embedded/spark.tar.gz"
+ dest="${project.build.directory}/embedded"
+ compression="gzip"
+ />
+ <move
+ todir="${project.build.directory}/embedded/spark" >
+ <fileset dir="${project.build.directory}/embedded/${spark.folder}" includes="**"/>
+ </move>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/assemblies/empty.xml b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/assemblies/empty.xml
new file mode 100644
index 0000000..35738b1
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/assemblies/empty.xml
@@ -0,0 +1,21 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id>empty</id>
+ <formats/>
+</assembly>
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala
index f9ed4b2..58efa97 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala
@@ -53,6 +53,12 @@ class AnomalyDetectionAppConfig extends Configuration {
@Valid
private val metricDefinitionDBConfiguration = new MetricDefinitionDBConfiguration
+ /**
+ * Spark configurations
+ */
+ @Valid
+ private val sparkConfiguration = new SparkConfiguration
+
/*
AMS HBase Conf
*/
@@ -76,4 +82,8 @@ class AnomalyDetectionAppConfig extends Configuration {
@JsonProperty("metricDefinitionDB")
def getMetricDefinitionDBConfiguration: MetricDefinitionDBConfiguration = metricDefinitionDBConfiguration
+
+ @JsonProperty("spark")
+ def getSparkConfiguration: SparkConfiguration = sparkConfiguration
+
}
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionServiceConfiguration.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionServiceConfiguration.scala
index b560713..a453f03 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionServiceConfiguration.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/MetricDefinitionServiceConfiguration.scala
@@ -17,8 +17,6 @@
package org.apache.ambari.metrics.adservice.configuration
-import javax.validation.constraints.NotNull
-
import com.fasterxml.jackson.annotation.JsonProperty
/**
@@ -26,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonProperty
*/
class MetricDefinitionServiceConfiguration {
- @NotNull
private val inputDefinitionDirectory: String = ""
@JsonProperty
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/SparkConfiguration.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/SparkConfiguration.scala
new file mode 100644
index 0000000..30efdc7
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/configuration/SparkConfiguration.scala
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.adservice.configuration
+
+import javax.validation.constraints.NotNull
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+class SparkConfiguration {
+
+ @NotNull
+ private var mode: String = _
+
+ @NotNull
+ private var masterHostPort: String = _
+
+ @JsonProperty
+ def getMode: String = mode
+
+ @JsonProperty
+ def getMasterHostPort: String = masterHostPort
+
+}
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/PhoenixAnomalyStoreAccessor.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/PhoenixAnomalyStoreAccessor.scala
index 147d1f7..53e6dee 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/PhoenixAnomalyStoreAccessor.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/db/PhoenixAnomalyStoreAccessor.scala
@@ -21,14 +21,11 @@ import java.sql.{Connection, PreparedStatement, ResultSet, SQLException}
import java.util.concurrent.TimeUnit.SECONDS
import org.apache.ambari.metrics.adservice.app.AnomalyDetectionAppConfig
-import org.apache.ambari.metrics.adservice.common._
import org.apache.ambari.metrics.adservice.configuration.HBaseConfiguration
import org.apache.ambari.metrics.adservice.metadata.{MetricDefinitionService, MetricKey}
import org.apache.ambari.metrics.adservice.model.AnomalyDetectionMethod.AnomalyDetectionMethod
import org.apache.ambari.metrics.adservice.model.AnomalyType.AnomalyType
-import org.apache.ambari.metrics.adservice.model.{AnomalyDetectionMethod, AnomalyType, MetricAnomalyInstance}
-import org.apache.ambari.metrics.adservice.subsystem.pointintime.PointInTimeAnomalyInstance
-import org.apache.ambari.metrics.adservice.subsystem.trend.TrendAnomalyInstance
+import org.apache.ambari.metrics.adservice.model._
import org.apache.hadoop.hbase.util.RetryCounterFactory
import org.slf4j.{Logger, LoggerFactory}
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/pointintime/PointInTimeAnomalyInstance.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/PointInTimeAnomalyInstance.scala
similarity index 90%
rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/pointintime/PointInTimeAnomalyInstance.scala
rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/PointInTimeAnomalyInstance.scala
index 56ca2c1..470cc2c 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/pointintime/PointInTimeAnomalyInstance.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/PointInTimeAnomalyInstance.scala
@@ -15,15 +15,13 @@
* limitations under the License.
*/
-package org.apache.ambari.metrics.adservice.subsystem.pointintime
+package org.apache.ambari.metrics.adservice.model
import java.util.Date
-import org.apache.ambari.metrics.adservice.common.Season
import org.apache.ambari.metrics.adservice.metadata.MetricKey
import org.apache.ambari.metrics.adservice.model.AnomalyDetectionMethod.AnomalyDetectionMethod
import org.apache.ambari.metrics.adservice.model.AnomalyType.AnomalyType
-import org.apache.ambari.metrics.adservice.model.{AnomalyType, MetricAnomalyInstance}
class PointInTimeAnomalyInstance(val metricKey: MetricKey,
val timestamp: Long,
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Range.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Range.scala
similarity index 96%
rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Range.scala
rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Range.scala
index 003c18f..4ad35e7 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Range.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Range.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.ambari.metrics.adservice.common
+package org.apache.ambari.metrics.adservice.model
/**
* Class to capture a Range in a Season.
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Season.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Season.scala
similarity index 96%
rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Season.scala
rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Season.scala
index f875e3b..84784bc 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Season.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/Season.scala
@@ -15,14 +15,14 @@
* limitations under the License.
*/
-package org.apache.ambari.metrics.adservice.common
+package org.apache.ambari.metrics.adservice.model
import java.time.DayOfWeek
import java.util.Calendar
import javax.xml.bind.annotation.XmlRootElement
-import org.apache.ambari.metrics.adservice.common.SeasonType.SeasonType
+import org.apache.ambari.metrics.adservice.model.SeasonType.SeasonType
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/SeasonType.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/SeasonType.scala
similarity index 94%
rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/SeasonType.scala
rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/SeasonType.scala
index 067972c..b510531 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/SeasonType.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/SeasonType.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.ambari.metrics.adservice.common
+package org.apache.ambari.metrics.adservice.model
object SeasonType extends Enumeration{
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/TimeRange.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TimeRange.scala
similarity index 96%
rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/TimeRange.scala
rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TimeRange.scala
index 50df658..0be2564 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/TimeRange.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TimeRange.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.ambari.metrics.adservice.common
+package org.apache.ambari.metrics.adservice.model
import java.util.Date
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TrendAnomalyInstance.scala
similarity index 90%
rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala
rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TrendAnomalyInstance.scala
index 7392d59..d67747c 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/subsystem/trend/TrendAnomalyInstance.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/model/TrendAnomalyInstance.scala
@@ -15,13 +15,11 @@
* limitations under the License.
*/
-package org.apache.ambari.metrics.adservice.subsystem.trend
+package org.apache.ambari.metrics.adservice.model
-import org.apache.ambari.metrics.adservice.common.{Season, TimeRange}
import org.apache.ambari.metrics.adservice.metadata.MetricKey
import org.apache.ambari.metrics.adservice.model.AnomalyDetectionMethod.AnomalyDetectionMethod
import org.apache.ambari.metrics.adservice.model.AnomalyType.AnomalyType
-import org.apache.ambari.metrics.adservice.model.{AnomalyType, MetricAnomalyInstance}
case class TrendAnomalyInstance (metricKey: MetricKey,
anomalousPeriod: TimeRange,
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/resources/config.yaml
similarity index 86%
rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml
rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/resources/config.yaml
index 7de06b4..6b09499 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/main/resources/config.yml
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/resources/config.yaml
@@ -10,16 +10,6 @@
#See the License for the specific language governing permissions and
#limitations under the License.
-server:
- applicationConnectors:
- - type: http
- port: 9999
- requestLog:
- type: external
-
-logging:
- type: external
-
metricDefinitionService:
inputDefinitionDirectory: /etc/ambari-metrics-anomaly-detection/conf/definitionDirectory
@@ -40,7 +30,6 @@ metricDefinitionDB:
# Path to Level DB directory
dbDirPath: /var/lib/ambari-metrics-anomaly-detection/
-#subsystemService:
-# spark:
-# pointInTime:
-# trend:
\ No newline at end of file
+spark:
+ mode: standalone
+ masterHostPort: localhost:7077
\ No newline at end of file
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala
index 989ba21..76391a0 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfigTest.scala
@@ -18,28 +18,31 @@
package org.apache.ambari.metrics.adservice.app
import java.io.File
+import java.net.URL
import javax.validation.Validator
import org.scalatest.FunSuite
import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.datatype.guava.GuavaModule
import io.dropwizard.configuration.YamlConfigurationFactory
-import io.dropwizard.jackson.Jackson
import io.dropwizard.jersey.validation.Validators
class AnomalyDetectionAppConfigTest extends FunSuite {
test("testConfiguration") {
- val objectMapper: ObjectMapper = Jackson.newObjectMapper()
+ val classLoader = getClass.getClassLoader
+ val url: URL = classLoader.getResource("config.yaml")
+ val file = new File(url.getFile)
+
+ val objectMapper: ObjectMapper = new ObjectMapper()
+ objectMapper.registerModule(new GuavaModule)
val validator: Validator = Validators.newValidator
val factory: YamlConfigurationFactory[AnomalyDetectionAppConfig] =
new YamlConfigurationFactory[AnomalyDetectionAppConfig](classOf[AnomalyDetectionAppConfig], validator, objectMapper, "")
-
- val classLoader = getClass.getClassLoader
- val file = new File(classLoader.getResource("config.yml").getFile)
val config = factory.build(file)
assert(config.isInstanceOf[AnomalyDetectionAppConfig])
@@ -48,17 +51,17 @@ class AnomalyDetectionAppConfigTest extends FunSuite {
"/etc/ambari-metrics-anomaly-detection/conf/definitionDirectory")
assert(config.getMetricCollectorConfiguration.getHosts == "host1,host2")
-
assert(config.getMetricCollectorConfiguration.getPort == "6188")
assert(config.getAdServiceConfiguration.getAnomalyDataTtl == 604800)
assert(config.getMetricDefinitionDBConfiguration.getDbDirPath == "/var/lib/ambari-metrics-anomaly-detection/")
-
assert(config.getMetricDefinitionDBConfiguration.getVerifyChecksums)
-
assert(!config.getMetricDefinitionDBConfiguration.getPerformParanoidChecks)
+ assert(config.getSparkConfiguration.getMode.equals("standalone"))
+ assert(config.getSparkConfiguration.getMasterHostPort.equals("localhost:7077"))
+
}
}
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala
index e38ea40..7330ff9 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/app/DefaultADResourceSpecTest.scala
@@ -36,7 +36,7 @@ class DefaultADResourceSpecTest extends FunSpec with Matchers {
describe("/anomaly") {
it("Must return default message") {
- withAppRunning(classOf[AnomalyDetectionApp], Resources.getResource("config.yml").getPath) { rule =>
+ withAppRunning(classOf[AnomalyDetectionApp], Resources.getResource("config.yaml").getPath) { rule =>
val json = client.target(s"http://localhost:${rule.getLocalPort}/anomaly")
.request().accept(APPLICATION_JSON).buildGet().invoke(classOf[String])
val dtf = java.time.format.DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm")
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/RangeTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/RangeTest.scala
similarity index 85%
rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/RangeTest.scala
rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/RangeTest.scala
index b610b97..16f4951 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/RangeTest.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/RangeTest.scala
@@ -15,14 +15,15 @@
* limitations under the License.
*/
-package org.apache.ambari.metrics.adservice.common
+package org.apache.ambari.metrics.adservice.model
+import org.apache.ambari.metrics.adservice.model
import org.scalatest.FlatSpec
class RangeTest extends FlatSpec {
"A Range " should " return true for inner and boundary values" in {
- val range : Range = Range(4,6)
+ val range : model.Range = model.Range(4,6)
assert(range.withinRange(5))
assert(range.withinRange(6))
assert(range.withinRange(4))
@@ -30,7 +31,7 @@ class RangeTest extends FlatSpec {
}
it should "accept same lower and higher range values" in {
- val range : Range = Range(4,4)
+ val range : model.Range = model.Range(4,4)
assert(range.withinRange(4))
}
diff --git a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/SeasonTest.scala b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/SeasonTest.scala
similarity index 75%
rename from ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/SeasonTest.scala
rename to ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/SeasonTest.scala
index a823c73..a661c05 100644
--- a/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/common/SeasonTest.scala
+++ b/ambari-metrics/ambari-metrics-anomaly-detection-service/src/test/scala/org/apache/ambari/metrics/adservice/model/SeasonTest.scala
@@ -15,10 +15,11 @@
* limitations under the License.
*/
-package org.apache.ambari.metrics.adservice.common
+package org.apache.ambari.metrics.adservice.model
import java.util.Calendar
+import org.apache.ambari.metrics.adservice.model
import org.scalatest.FunSuite
class SeasonTest extends FunSuite {
@@ -26,7 +27,7 @@ class SeasonTest extends FunSuite {
test("testBelongsTo") {
//Create Season for weekdays. Mon to Friday and 9AM - 5PM
- var season : Season = Season(Range(Calendar.MONDAY,Calendar.FRIDAY), Range(9,17))
+ var season : Season = Season(model.Range(Calendar.MONDAY,Calendar.FRIDAY), model.Range(9,17))
//Try with a timestamp on a Monday, @ 9AM.
val c = Calendar.getInstance
@@ -41,7 +42,7 @@ class SeasonTest extends FunSuite {
assert(!season.belongsTo(c.getTimeInMillis))
//Create Season for Monday 11AM - 12Noon.
- season = Season(Range(Calendar.MONDAY,Calendar.MONDAY), Range(11,12))
+ season = Season(model.Range(Calendar.MONDAY,Calendar.MONDAY), model.Range(11,12))
c.set(2017, Calendar.OCTOBER, 30, 9, 0, 0)
assert(!season.belongsTo(c.getTimeInMillis))
@@ -50,7 +51,7 @@ class SeasonTest extends FunSuite {
//Create Season from Friday to Monday and 9AM - 5PM
- season = Season(Range(Calendar.FRIDAY,Calendar.MONDAY), Range(9,17))
+ season = Season(model.Range(Calendar.FRIDAY,Calendar.MONDAY), model.Range(9,17))
//Try with a timestamp on a Monday, @ 9AM.
c.set(2017, Calendar.OCTOBER, 30, 9, 0, 0)
@@ -67,23 +68,23 @@ class SeasonTest extends FunSuite {
test("testEquals") {
- var season1: Season = Season(Range(4,5), Range(2,3))
- var season2: Season = Season(Range(4,5), Range(2,3))
+ var season1: Season = Season(model.Range(4,5), model.Range(2,3))
+ var season2: Season = Season(model.Range(4,5), model.Range(2,3))
assert(season1 == season2)
- var season3: Season = Season(Range(4,4), Range(2,3))
+ var season3: Season = Season(model.Range(4,4), model.Range(2,3))
assert(!(season1 == season3))
}
test("testSerialize") {
- val season1 : Season = Season(Range(Calendar.MONDAY,Calendar.FRIDAY), Range(9,17))
+ val season1 : Season = Season(model.Range(Calendar.MONDAY,Calendar.FRIDAY), model.Range(9,17))
val seasonString = Season.toJson(season1)
val season2 : Season = Season.fromJson(seasonString)
assert(season1 == season2)
- val season3 : Season = Season(Range(Calendar.MONDAY,Calendar.THURSDAY), Range(9,17))
+ val season3 : Season = Season(model.Range(Calendar.MONDAY,Calendar.THURSDAY), model.Range(9,17))
assert(!(season2 == season3))
}
diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml
index 43ff285..b1a6430 100644
--- a/ambari-metrics/ambari-metrics-assembly/pom.xml
+++ b/ambari-metrics/ambari-metrics-assembly/pom.xml
@@ -42,6 +42,7 @@
<storm-sink-legacy.dir>${project.basedir}/../ambari-metrics-storm-sink-legacy</storm-sink-legacy.dir>
<flume-sink.dir>${project.basedir}/../ambari-metrics-flume-sink</flume-sink.dir>
<kafka-sink.dir>${project.basedir}/../ambari-metrics-kafka-sink</kafka-sink.dir>
+ <anomaly-detection.dir>${project.basedir}/../ambari-metrics-anomaly-detection-service</anomaly-detection.dir>
<python.ver>python >= 2.6</python.ver>
<python.devel>python-devel</python.devel>
<deb.publisher>Apache</deb.publisher>
@@ -56,6 +57,7 @@
<storm.sink.legacy.jar>ambari-metrics-storm-sink-legacy-with-common-${project.version}.jar</storm.sink.legacy.jar>
<flume.sink.jar>ambari-metrics-flume-sink-with-common-${project.version}.jar</flume.sink.jar>
<kafka.sink.jar>ambari-metrics-kafka-sink-with-common-${project.version}.jar</kafka.sink.jar>
+ <anomaly.detection.jar>ambari-metrics-anomaly-detection-service-${project.version}.jar</anomaly.detection.jar>
</properties>
<build>
@@ -139,6 +141,22 @@
</configuration>
</execution>
<execution>
+ <id>anomaly-detection</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <attach>false</attach>
+ <finalName>ambari-metrics-anomaly-detection-${project.version}</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>${assemblydescriptor.anomaly-detection}</descriptor>
+ </descriptors>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ </configuration>
+ </execution>
+ <execution>
<id>hadoop-sink</id>
<phase>prepare-package</phase>
<goals>
@@ -638,6 +656,81 @@
</configuration>
</execution>
+ <!--ambari-metrics-anomaly-detection-->
+ <execution>
+ <id>ambari-metrics-anomaly-detection</id>
+ <phase>package</phase>
+ <goals>
+ <goal>rpm</goal>
+ </goals>
+ <configuration>
+ <name>ambari-metrics-anomaly-detection</name>
+ <copyright>2012, Apache Software Foundation</copyright>
+ <group>Development</group>
+ <description>Maven Recipe: RPM Package.</description>
+ <autoRequires>false</autoRequires>
+
+
+ <defaultFilemode>644</defaultFilemode>
+ <defaultDirmode>755</defaultDirmode>
+ <defaultUsername>root</defaultUsername>
+ <defaultGroupname>root</defaultGroupname>
+
+ <postinstallScriptlet>
+ <scriptFile>${project.build.directory}/resources/rpm/anomaly-detection/postinstall.sh</scriptFile>
+ <fileEncoding>utf-8</fileEncoding>
+ </postinstallScriptlet>
+
+ <mappings>
+ <mapping>
+ <!--jars-->
+ <directory>/usr/lib/ambari-metrics-anomaly-detection/</directory>
+ <sources>
+ <source>
+ <location>
+ ${anomaly-detection.dir}/target/ambari-metrics-anomaly-detection-service-${project.version}.jar
+ </location>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
+ <directory>/usr/lib/ambari-metrics-anomaly-detection/spark</directory>
+ <sources>
+ <source>
+ <location>
+ ${anomaly-detection.dir}/target/embedded/spark
+ </location>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
+ <directory>/usr/sbin</directory>
+ <filemode>755</filemode>
+ <username>root</username>
+ <groupname>root</groupname>
+ <directoryIncluded>false</directoryIncluded>
+ <sources>
+ <source>
+ <location>${anomaly-detection.dir}/conf/unix/ambari-metrics-admanager</location>
+ <filter>false</filter>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
+ <directory>/etc/ambari-metrics-anomaly-detection/conf</directory>
+ <configuration>true</configuration>
+ <sources>
+ <source>
+ <location>${anomaly-detection.dir}/conf/unix/config.yaml</location>
+ </source>
+ <source>
+ <location>${anomaly-detection.dir}/conf/unix/log4j.properties</location>
+ </source>
+ </sources>
+ </mapping>
+ </mappings>
+ </configuration>
+ </execution>
</executions>
</plugin>
@@ -757,10 +850,13 @@
<path>/etc/ambari-metrics-collector/conf</path>
<path>/etc/ambari-metrics-grafana/conf</path>
<path>/etc/ams-hbase/conf</path>
+ <path>/etc/ambari-metrics-anomaly-detection/conf</path>
<path>/var/run/ams-hbase</path>
<path>/var/run/ambari-metrics-grafana</path>
<path>/var/log/ambari-metrics-grafana</path>
<path>/var/lib/ambari-metrics-collector</path>
+ <path>/usr/lib/ambari-metrics-anomaly-detection</path>
+ <path>/var/lib/ambari-metrics-anomaly-detection</path>
<path>/var/lib/ambari-metrics-monitor/lib</path>
<path>/var/lib/ambari-metrics-grafana</path>
<path>/usr/lib/ambari-metrics-hadoop-sink</path>
@@ -979,6 +1075,49 @@
</mapper>
</data>
+ <!-- Anomaly Detection -->
+ <data>
+ <src>${anomaly-detection.dir}/target/${anomaly.detection.jar}</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <dirmode>644</dirmode>
+ <prefix>/usr/lib/ambari-metrics-anomaly-detection</prefix>
+ </mapper>
+ </data>
+ <data>
+ <type>link</type>
+ <linkName>/usr/lib/ambari-metrics-anomaly-detection/ambari-metrics-anomaly-detection-service.jar</linkName>
+ <linkTarget>/usr/lib/ambari-metrics-anomaly-detection/${anomaly.detection.jar}</linkTarget>
+ <symlink>true</symlink>
+ </data>
+ <data>
+ <src>${anomaly-detection.dir}/target/embedded/spark</src>
+ <type>directory</type>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/lib/ambari-metrics-anomaly-detection/spark</prefix>
+ <filemode>644</filemode>
+ </mapper>
+ </data>
+ <data>
+ <src>${anomaly-detection.dir}/conf/unix/config.yaml</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <filemode>755</filemode>
+ <prefix>/etc/ambari-metrics-anomaly-detection/conf</prefix>
+ </mapper>
+ </data>
+ <data>
+ <src>${anomaly-detection.dir}/conf/unix/log4j.properties</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <filemode>755</filemode>
+ <prefix>/etc/ambari-metrics-anomaly-detection/conf</prefix>
+ </mapper>
+ </data>
<!-- hadoop sink -->
<data>
@@ -1075,6 +1214,8 @@
<assemblydescriptor.monitor>src/main/assembly/monitor.xml</assemblydescriptor.monitor>
<assemblydescriptor.sink>src/main/assembly/sink.xml</assemblydescriptor.sink>
<assemblydescriptor.grafana>src/main/assembly/grafana.xml</assemblydescriptor.grafana>
+ <assemblydescriptor.anomaly-detection>src/main/assembly/anomaly-detection.xml</assemblydescriptor.anomaly-detection>
+
<packagingFormat>jar</packagingFormat>
</properties>
<build>
@@ -1354,6 +1495,13 @@
<artifactId>ambari-metrics-host-aggregator</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-metrics-anomaly-detection-service</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <optional>true</optional>
+ </dependency>
</dependencies>
diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/anomaly-detection.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/anomaly-detection.xml
new file mode 100644
index 0000000..b05aaf3
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/anomaly-detection.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd">
+ <id>anomaly-detection</id>
+ <formats>
+ <format>dir</format>
+ <format>tar.gz</format>
+ </formats>
+
+ <fileSets>
+ <fileSet>
+ <directory>${anomaly-detection.dir}/target/embedded/spark</directory>
+ <outputDirectory>anomaly-detection/spark</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${anomaly-detection.dir}/conf/unix</directory>
+ <outputDirectory>anomaly-detection/bin</outputDirectory>
+ <includes>
+ <include>ambari-metrics-admanager</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${anomaly-detection.dir}/conf/unix</directory>
+ <outputDirectory>anomaly-detection/conf</outputDirectory>
+ <includes>
+ <include>config.yaml</include>
+ <include>log4j.properties</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <files>
+ <file>
+ <fileMode>644</fileMode>
+ <source>${anomaly-detection.dir}/target/ambari-metrics-anomaly-detection-service-${project.version}.jar
+ </source>
+ <outputDirectory>anomaly-detection</outputDirectory>
+ </file>
+ </files>
+</assembly>
\ No newline at end of file
diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/anomaly-detection/postinstall.sh b/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/anomaly-detection/postinstall.sh
new file mode 100644
index 0000000..399c439
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/anomaly-detection/postinstall.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+
+ANOMALY_DETECTION_LINK_NAME="/usr/lib/ambari-metrics-anomaly-detection/ambari-metrics-anomaly-detection-service.jar"
+ANOMALY_DETECTION_JAR="/usr/lib/ambari-metrics-anomaly-detection/${anomaly.detection.jar}"
+
+JARS=(${ANOMALY_DETECTION_JAR})
+LINKS=(${ANOMALY_DETECTION_LINK_NAME})
+
+for index in ${!LINKS[*]}
+do
+ rm -f ${LINKS[$index]} ; ln -s ${JARS[$index]} ${LINKS[$index]}
+done
diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml
index a8c71e6..98559e6 100644
--- a/ambari-metrics/pom.xml
+++ b/ambari-metrics/pom.xml
@@ -32,9 +32,9 @@
<module>ambari-metrics-timelineservice</module>
<module>ambari-metrics-host-monitoring</module>
<module>ambari-metrics-grafana</module>
- <module>ambari-metrics-assembly</module>
<module>ambari-metrics-host-aggregator</module>
<module>ambari-metrics-anomaly-detection-service</module>
+ <module>ambari-metrics-assembly</module>
</modules>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -53,6 +53,8 @@
<grafana.tar>https://grafanarel.s3.amazonaws.com/builds/grafana-2.6.0.linux-x64.tar.gz</grafana.tar>
<phoenix.tar>https://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.4.0/tars/phoenix/phoenix-4.7.0.2.6.4.0-91.tar.gz</phoenix.tar>
<phoenix.folder>phoenix-4.7.0.2.6.4.0-91</phoenix.folder>
+ <spark.tar>http://dev.hortonworks.com.s3.amazonaws.com/HDP/centos7/3.x/BUILDS/3.0.0.0-439/tars/spark2/spark-2.1.0.3.0.0.0-439-bin-3.0.0.3.0.0.0-439.tgz</spark.tar>
+ <spark.folder>spark-2.1.0.3.0.0.0-439-bin-3.0.0.3.0.0.0-439</spark.folder>
<resmonitor.install.dir>
/usr/lib/python2.6/site-packages/resource_monitoring
</resmonitor.install.dir>
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml
index 2c6bbf7..9862f10 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-config.xml
@@ -102,6 +102,10 @@
performParanoidChecks: false
# Path to Level DB directory
dbDirPath: {{ams_ad_data_dir}}
+
+ spark:
+ mode: {{admanager_spark_op_mode}}
+ masterHostPort: {{admanager_spark_hostport}}
</value>
<value-attributes>
<type>content</type>
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml
index a79796b..91073ee 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-env.xml
@@ -83,10 +83,10 @@
export AMS_AD_HEAPSIZE={{ams_admanager_heapsize}}
# Anomaly Detection Manager data dir
- export AMS_AD_DATA_DIR={{ams_admanager_data_dir}}
+ export AMS_AD_DATA_DIR={{ams_ad_data_dir}}
# Anomaly Detection Manager options
- export AMS_AD_OPTS="
+ export AMS_AD_OPTS=$AMS_AD_OPTS
{% if security_enabled %}
export AMS_AD_OPTS="$AMS_AD_OPTS -Djava.security.auth.login.config={{ams_ad_jaas_config_file}}"
{% endif %}
@@ -95,6 +95,8 @@
export AMS_AD_GC_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{ams_ad_log_dir}}/admanager-gc.log-`date +'%Y%m%d%H%M'`"
export AMS_AD_OPTS="$AMS_AD_OPTS $AMS_AD_GC_OPTS"
+ # Anomaly Detection Manager data dir
+ export AMS_AD_STANDALONE_SPARK_ENABLED={{ams_ad_standalone_spark_enabled}}
</value>
<value-attributes>
<type>content</type>
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml
index b1f821e..ad28dcd 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-log4j.xml
@@ -63,7 +63,7 @@
#
# Define some default values that can be overridden by system properties
- ams.ad.log.dir=.
+ ams.ad.log.dir={{ams_ad_log_dir}}
ams.ad.log.file=ambari-metrics-admanager.log
# Root logger option
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-spark-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-spark-env.xml
new file mode 100644
index 0000000..3c2fb89
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-admanager-spark-env.xml
@@ -0,0 +1,129 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <property>
+ <name>spark_daemon_memory</name>
+ <value>512</value>
+ <description>Memory for Master, Worker and history server (default: 1G)</description>
+ <value-attributes>
+ <type>int</type>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spark_master_port</name>
+ <value>6190</value>
+ <description>Start the master on this port</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spark_master_webui_port</name>
+ <value>6180</value>
+ <description>Port for the master web UI</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spark_worker_cores</name>
+ <value>4</value>
+ <description>Total number of cores to allow Spark applications to use on the machine</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spark_worker_memory</name>
+ <value>2048</value>
+ <description>Total amount of memory to allow Spark applications to use on the machine</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spark_worker_webui_port</name>
+ <value>6181</value>
+ <description>Port for the worker web UI</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for spark-env.sh file</description>
+ <value>
+ #!/usr/bin/env bash
+
+ # This file is sourced when running various Spark programs.
+ # Copy it as spark-env.sh and edit that to configure Spark for your site.
+
+ # Generic options for the daemons used in the standalone deploy mode
+
+ export SPARK_MASTER_HOST={{hostname}}
+ export SPARK_MASTER_PORT={{spark_master_port}}
+ export SPARK_MASTER_WEBUI_PORT={{spark_master_webui_port}}
+ export SPARK_WORKER_CORES={{spark_worker_cores}}
+ export SPARK_WORKER_MEMORY={{spark_worker_memory}}m
+ export SPARK_WORKER_WEBUI_PORT={{spark_worker_webui_port}}
+ export SPARK_WORKER_DIR={{ams_ad_log_dir}}
+
+ export SPARK_MASTER_OPTS=$SPARK_MASTER_OPTS
+ export SPARK_WORKER_OPTS=$SPARK_WORKER_OPTS
+
+ export SPARK_MASTER_PORT={{spark_master_port}}
+ # Alternate conf dir. (Default: ${SPARK_HOME}/conf)
+ export SPARK_CONF_DIR={{ams_ad_conf_dir}}
+
+ # Where log files are stored.(Default:${SPARK_HOME}/logs)
+ export SPARK_LOG_DIR={{ams_ad_log_dir}}
+
+ # Where the pid file is stored. (Default: /tmp)
+ export SPARK_PID_DIR={{ams_ad_pid_dir}}
+
+ #Memory for Master, Worker and history server (default: 1024MB)
+ export SPARK_DAEMON_MEMORY={{spark_daemon_memory}}m
+
+ # A string representing this instance of spark.(Default: $USER)
+ SPARK_IDENT_STRING=$USER
+
+ # The scheduling priority for daemons. (Default: 0)
+ SPARK_NICENESS=0
+
+ # Options read in YARN client mode
+ #SPARK_EXECUTOR_INSTANCES="2" #Number of workers to start (Default: 2)
+ #SPARK_EXECUTOR_CORES="1" #Number of cores for the workers (Default: 1).
+ #SPARK_EXECUTOR_MEMORY="1G" #Memory per Worker (e.g. 1000M, 2G) (Default: 1G)
+ #SPARK_DRIVER_MEMORY="512M" #Memory for Master (e.g. 1000M, 2G) (Default: 512 Mb)
+ #SPARK_YARN_APP_NAME="spark" #The name of your application (Default: Spark)
+ #SPARK_YARN_QUEUE="default" #The hadoop queue to use for allocation requests (Default: default)
+ #SPARK_YARN_DIST_FILES="" #Comma separated list of files to be distributed with the job.
+ #SPARK_YARN_DIST_ARCHIVES="" #Comma separated list of archives to be distributed with the job.
+
+ #export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+ #export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}}
+
+ # The java implementation to use.
+ export JAVA_HOME={{java_home}}
+
+ #HDP Version
+ export HDP_VERSION=3.0.0
+
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
index 41e278d..bcf6268 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
@@ -138,6 +138,7 @@
<config-type>ams-admanager-config</config-type>
<config-type>ams-admanager-env</config-type>
<config-type>ams-admanager-log4j</config-type>
+ <config-type>ams-admanager-spark-env</config-type>
</configuration-dependencies>
<logs>
<log>
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
index 7ab0547..4c6951a 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
@@ -555,14 +555,26 @@ def ams(name=None, action=None):
if (params.ams_ad_log4j_props != None):
File(os.path.join(params.ams_ad_conf_dir, "log4j.properties"),
owner=params.ams_user,
- content=params.ams_ad_log4j_props
+ content=InlineTemplate(params.ams_ad_log4j_props)
)
+ File(format("{ams_ad_conf_dir}/ams-admanager-spark-env.sh"),
+ owner=params.ams_user,
+ group=params.user_group,
+ content=InlineTemplate(params.ams_ad_spark_env_sh_template)
+ )
+
if action != 'stop':
for dir in ams_ad_directories:
Execute(('chown', '-R', params.ams_user, dir),
sudo=True
)
+ Execute(('chmod', '-R', '755', format("{ams_admanager_lib_dir}/spark/bin")),
+ sudo = True,
+ )
+ Execute(('chmod', '-R', '755', format("{ams_admanager_lib_dir}/spark/sbin")),
+ sudo = True,
+ )
def is_spnego_enabled(params):
if 'core-site' in params.config['configurations'] \
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py
index 96c4454..33c8832 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams_admanager.py
@@ -45,7 +45,7 @@ class AmsADManager(Script):
Execute(start_cmd,
user=params.ams_user
)
- pidfile = format("{ams_ad_pid_dir}/admanager.pid")
+ pidfile = format("{ams_ad_pid_dir}/ambari-metrics-admanager.pid")
if not sudo.path_exists(pidfile):
Logger.warning("Pid file doesn't exist after starting of the component.")
else:
@@ -57,7 +57,7 @@ class AmsADManager(Script):
env.set_params(params)
self.configure(env, action = 'stop')
Execute((format("{ams_admanager_script}"), 'stop'),
- sudo=True
+ user=params.ams_user
)
def status(self, env):
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 40d3db6..dd2a686 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -154,6 +154,7 @@ def get_ambari_version():
pass
return ambari_version
+hostname = config['agentLevelParams']['hostname']
ams_collector_log_dir = config['configurations']['ams-env']['metrics_collector_log_dir']
ams_collector_conf_dir = "/etc/ambari-metrics-collector/conf"
@@ -199,6 +200,26 @@ ams_admanager_config_template = config['configurations']['ams-admanager-config']
ams_admanager_script = "/usr/sbin/ambari-metrics-admanager"
ams_admanager_port = config['configurations']['ams-admanager-config']['ambari.metrics.admanager.application.port']
ams_admanager_heapsize = config['configurations']['ams-admanager-env']['ams_admanager_heapsize']
+ams_admanager_lib_dir = "/usr/lib/ambari-metrics-anomaly-detection"
+ams_admanager_jar = format("{ams_admanager_lib_dir}/ambari-metrics-anomaly-detection-service-*.jar")
+ams_ad_log_max_backup_size = default('configurations/ams-admanager-log4j/ams_ad_log_max_backup_size',80)
+ams_ad_log_number_of_backup_files = default('configurations/ams-admanager-log4j/ams_ad_log_number_of_backup_files',60)
+
+admanager_spark_op_mode = config['configurations']['ams-admanager-config']['ambari.metrics.admanager.spark.operation.mode']
+ams_ad_spark_env_sh_template = config['configurations']['ams-admanager-spark-env']['content']
+spark_master_port = default("/configurations/ams-admanager-spark-env/spark_master_port", 6190)
+spark_master_webui_port = default("/configurations/ams-admanager-spark-env/spark_master_webui_port", 6180)
+spark_worker_cores = default("/configurations/ams-admanager-spark-env/spark_worker_cores", 4)
+spark_worker_memory = default("/configurations/ams-admanager-spark-env/spark_worker_memory", 2048)
+spark_worker_webui_port = default("/configurations/ams-admanager-spark-env/spark_worker_webui_port", 6181)
+spark_daemon_memory = default("/configurations/ams-admanager-spark-env/spark_daemon_memory", 1024)
+
+if admanager_spark_op_mode == 'spark-on-yarn':
+ admanager_spark_hostport = hostname + ":" + spark_master_port #TODO : Fix for spark on yarn mode.
+ ams_ad_standalone_spark_enabled = False
+else:
+ admanager_spark_hostport = hostname + ":" + spark_master_port
+ ams_ad_standalone_spark_enabled = True
if (('ams-admanager-log4j' in config['configurations']) and ('content' in config['configurations']['ams-admanager-log4j'])):
ams_ad_log4j_props = config['configurations']['ams-admanager-log4j']['content']
@@ -289,7 +310,6 @@ else:
hbase_heapsize = master_heapsize
max_open_files_limit = default("/configurations/ams-hbase-env/max_open_files_limit", "32768")
-hostname = config['agentLevelParams']['hostname']
cluster_zookeeper_quorum_hosts = ",".join(config['clusterHostInfo']['zookeeper_server_hosts'])
if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py
index 3373592..347c290 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/status_params.py
@@ -37,7 +37,7 @@ ams_ad_pid_dir = config['configurations']['ams-admanager-env']['ams_ad_pid_dir']
monitor_pid_file = format("{ams_monitor_pid_dir}/ambari-metrics-monitor.pid")
grafana_pid_file = format("{ams_grafana_pid_dir}/grafana-server.pid")
-ams_ad_pid_file = format("{ams_ad_pid_dir}/admanager.pid")
+ams_ad_pid_file = format("{ams_ad_pid_dir}/ambari-metrics-admanager.pid")
security_enabled = config['configurations']['cluster-env']['security_enabled']
ams_hbase_conf_dir = format("{hbase_conf_dir}")
--
To stop receiving notification emails like this one, please contact
avijayan@apache.org.