You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2016/09/26 14:05:10 UTC
[7/7] incubator-metron git commit: METRON-427 Create Ambari
Management Pack for Metron Installation closes apache/incubator-metron#266
METRON-427 Create Ambari Management Pack for Metron Installation closes apache/incubator-metron#266
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/125dbef1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/125dbef1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/125dbef1
Branch: refs/heads/master
Commit: 125dbef1e59ff808a62e4f5a7d265aafbcf6bf08
Parents: 6834e14
Author: justinleet <ju...@gmail.com>
Authored: Mon Sep 26 09:53:37 2016 -0400
Committer: justinjleet <ju...@gmail.com>
Committed: Mon Sep 26 09:53:37 2016 -0400
----------------------------------------------------------------------
.gitignore | 1 +
metron-deployment/README.md | 63 +-
.../packaging/ambari/metron-mpack/pom.xml | 111 ++
.../src/main/assemblies/metron-mpack.xml | 43 +
.../ELASTICSEARCH/2.3.3/metainfo.xml | 29 +
.../addon-services/KIBANA/4.5.1/metainfo.xml | 33 +
.../KIBANA/4.5.1/quicklinks/quicklinks.json | 27 +
.../METRON/0.2.0BETA/metainfo.xml | 26 +
.../METRON/0.2.0BETA/role_command_order.json | 14 +
.../2.3.3/configuration/elastic-env.xml | 58 +
.../2.3.3/configuration/elastic-site.xml | 180 ++
.../2.3.3/configuration/elastic-sysconfig.xml | 98 ++
.../ELASTICSEARCH/2.3.3/metainfo.xml | 73 +
.../2.3.3/package/scripts/elastic.py | 68 +
.../2.3.3/package/scripts/elastic_master.py | 81 +
.../2.3.3/package/scripts/elastic_slave.py | 77 +
.../2.3.3/package/scripts/params.py | 77 +
.../2.3.3/package/scripts/properties_config.py | 34 +
.../2.3.3/package/scripts/service_check.py | 80 +
.../2.3.3/package/scripts/slave.py | 65 +
.../2.3.3/package/scripts/status_params.py | 26 +
.../templates/elasticsearch.master.yaml.j2 | 84 +
.../templates/elasticsearch.slave.yaml.j2 | 84 +
.../ELASTICSEARCH/2.3.3/role_command_order.json | 8 +
.../KIBANA/4.5.1/configuration/kibana-env.xml | 54 +
.../KIBANA/4.5.1/configuration/kibana-site.xml | 112 ++
.../common-services/KIBANA/4.5.1/metainfo.xml | 75 +
.../4.5.1/package/scripts/dashboard/__init__.py | 16 +
.../4.5.1/package/scripts/dashboard/dashboard.p | 1539 ++++++++++++++++++
.../package/scripts/dashboard/dashboardindex.py | 95 ++
.../4.5.1/package/scripts/kibana_master.py | 137 ++
.../KIBANA/4.5.1/package/scripts/params.py | 47 +
.../KIBANA/4.5.1/quicklinks/quicklinks.json | 28 +
.../0.2.0BETA/configuration/metron-env.xml | 174 ++
.../METRON/0.2.0BETA/metainfo.xml | 273 ++++
.../0.2.0BETA/package/files/addMysqlUser.sh | 44 +
.../0.2.0BETA/package/files/bro_index.template | 217 +++
.../0.2.0BETA/package/files/createMysqlGeoIp.sh | 42 +
.../0.2.0BETA/package/files/removeMysqlUser.sh | 33 +
.../package/files/snort_index.template | 183 +++
.../0.2.0BETA/package/files/yaf_index.template | 205 +++
.../package/scripts/enrichment_commands.py | 161 ++
.../package/scripts/enrichment_master.py | 87 +
.../package/scripts/indexing_commands.py | 142 ++
.../package/scripts/indexing_master.py | 120 ++
.../0.2.0BETA/package/scripts/metron_service.py | 76 +
.../0.2.0BETA/package/scripts/mysql_server.py | 61 +
.../0.2.0BETA/package/scripts/mysql_service.py | 46 +
.../0.2.0BETA/package/scripts/mysql_users.py | 58 +
.../0.2.0BETA/package/scripts/mysql_utils.py | 50 +
.../package/scripts/params/__init__.py | 18 +
.../0.2.0BETA/package/scripts/params/params.py | 32 +
.../package/scripts/params/params_linux.py | 169 ++
.../package/scripts/params/params_windows.py | 20 +
.../package/scripts/params/status_params.py | 57 +
.../package/scripts/parser_commands.py | 208 +++
.../0.2.0BETA/package/scripts/parser_master.py | 89 +
.../0.2.0BETA/package/scripts/service_check.py | 41 +
.../package/templates/enrichment.properties.j2 | 88 +
.../METRON/0.2.0BETA/quicklinks/quicklinks.json | 28 +
.../metron-mpack/src/main/resources/mpack.json | 85 +
.../2.3.3/configuration/elastic-env.xml | 58 -
.../2.3.3/configuration/elastic-site.xml | 180 --
.../2.3.3/configuration/elastic-sysconfig.xml | 98 --
.../ELASTICSEARCH/2.3.3/metainfo.xml | 73 -
.../2.3.3/package/scripts/elastic.py | 68 -
.../2.3.3/package/scripts/elastic_master.py | 79 -
.../2.3.3/package/scripts/elastic_slave.py | 76 -
.../2.3.3/package/scripts/params.py | 77 -
.../2.3.3/package/scripts/properties_config.py | 34 -
.../2.3.3/package/scripts/service_check.py | 80 -
.../2.3.3/package/scripts/slave.py | 65 -
.../2.3.3/package/scripts/status_params.py | 26 -
.../templates/elasticsearch.master.yaml.j2 | 84 -
.../templates/elasticsearch.slave.yaml.j2 | 84 -
.../ELASTICSEARCH/2.3.3/role_command_order.json | 8 -
.../0.2.0BETA/configuration/metron-indexing.xml | 48 -
.../INDEXING/0.2.0BETA/metainfo.xml | 105 --
.../0.2.0BETA/package/scripts/commands.py | 146 --
.../package/scripts/indexing_master.py | 71 -
.../0.2.0BETA/package/scripts/params.py | 29 -
.../0.2.0BETA/package/scripts/params_linux.py | 70 -
.../0.2.0BETA/package/scripts/params_windows.py | 17 -
.../0.2.0BETA/package/scripts/service_check.py | 37 -
.../0.2.0BETA/package/scripts/status_params.py | 19 -
.../0.2.0BETA/quicklinks/quicklinks.json | 28 -
.../KIBANA/4.5.1/configuration/kibana-env.xml | 54 -
.../KIBANA/4.5.1/configuration/kibana-site.xml | 112 --
.../common-services/KIBANA/4.5.1/metainfo.xml | 75 -
.../4.5.1/package/scripts/dashboard/__init__.py | 16 -
.../4.5.1/package/scripts/dashboard/dashboard.p | 1539 ------------------
.../package/scripts/dashboard/dashboardindex.py | 95 --
.../4.5.1/package/scripts/kibana_master.py | 146 --
.../KIBANA/4.5.1/package/scripts/params.py | 47 -
.../KIBANA/4.5.1/quicklinks/quicklinks.json | 28 -
.../0.2.0BETA/configuration/metron-parsers.xml | 57 -
.../PARSERS/0.2.0BETA/metainfo.xml | 108 --
.../0.2.0BETA/package/scripts/commands.py | 235 ---
.../package/scripts/params/__init__.py | 18 -
.../0.2.0BETA/package/scripts/params/params.py | 32 -
.../package/scripts/params/params_linux.py | 114 --
.../package/scripts/params/params_windows.py | 20 -
.../package/scripts/params/status_params.py | 22 -
.../0.2.0BETA/package/scripts/parser_master.py | 77 -
.../0.2.0BETA/package/scripts/service_check.py | 40 -
.../package/templates/metron-global.json | 1 -
.../0.2.0BETA/quicklinks/quicklinks.json | 28 -
.../PARSERS/0.2.0BETA/service_advisor.py | 68 -
.../HDP/2.3/services/PARSERS/metainfo.xml | 27 -
.../services/PARSERS/role_command_order.json | 17 -
.../HDP/2.4/services/ELASTICSEARCH/metainfo.xml | 41 -
.../HDP/2.4/services/INDEXING/metainfo.xml | 26 -
.../services/INDEXING/role_command_order.json | 9 -
.../stacks/HDP/2.4/services/KIBANA/metainfo.xml | 29 -
.../docker/rpm-docker/SPECS/metron.spec | 2 +
.../packaging/docker/rpm-docker/pom.xml | 218 +++
metron-deployment/pom.xml | 181 +-
.../roles/ambari_common/defaults/main.yml | 2 +-
.../vars/multi_vagrant_cluster.yml | 2 +-
.../roles/ambari_config/vars/single_node_vm.yml | 2 +-
.../roles/ambari_config/vars/small_cluster.yml | 2 +-
.../src/main/assembly/assembly.xml | 12 +
.../enrichment/adapters/geo/GeoAdapter.java | 5 +-
.../simplehbase/SimpleHBaseAdapter.java | 1 +
.../adapters/stellar/StellarAdapter.java | 8 +-
.../threatintel/ThreatIntelAdapter.java | 5 +-
.../apache/metron/enrichment/bolt/JoinBolt.java | 1 +
.../enrichment/bolt/ThreatIntelJoinBolt.java | 1 +
.../src/main/resources/ddl/geoip_ddl.sql | 49 +
.../writer/bolt/BulkMessageWriterBolt.java | 1 +
pom.xml | 4 +-
131 files changed, 6671 insertions(+), 4916 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 49e1afa..5e570e4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,3 +20,4 @@ tmp/**/*
temp/**
temp/**/*
+repodata/
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/README.md b/metron-deployment/README.md
index 6a44596..eceeb80 100644
--- a/metron-deployment/README.md
+++ b/metron-deployment/README.md
@@ -3,6 +3,8 @@ This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster, Me
playbooks. These playbooks currently only target RHEL/CentOS 6.x operating
systems.
+In addition, an Ambari Management Pack can be built which can be deployed in conjuction with RPMs detailed in this README.
+
## Prerequisites
The following tools are required to run these scripts:
@@ -27,7 +29,7 @@ structure can be found at http://docs.ansible.com/ansible/playbooks_best_practic
## Ambari
The Ambari playbook will install a Hadoop cluster with all the services and configuration required by Metron. This
-section can be skipped if installing Metron on a pre-existing cluster.
+section can be skipped if installing Metron on a pre-existing cluster.
Currently, this playbook supports building a local development cluster running on one node but options for other types
of clusters will be added in the future.
@@ -91,6 +93,62 @@ installed
Navigate to `incubator-metron/metron-deployment/vagrant/full-dev-platform` and run `vagrant up`. This also provides a good
example of how to run a full end-to-end Metron install.
+## Ambari Management Pack
+An Ambari Management Pack can be built in order to make the Metron service available on top of an existing stack, rather than needing a direct stack update.
+
+This will set up
+- Metron Parsers
+- Enrichment
+- Indexing
+- GeoIP database on MySQL
+- Optional Elasticsearch
+- Optional Kibana
+
+### Prerequisites
+- A cluster managed by Ambari 2.4
+- Metron RPMs available on the cluster in the /localrepo directory. See [RPM](#RPM) for further information.
+
+### Building Management Pack
+From `metron-deployment` run
+```
+mvn clean package
+```
+
+A tar.gz that can be used with Ambari can be found at `metron-deployment/packaging/ambari/metron-mpack/target/`
+
+### Installing Management Pack
+Before installing the mpack, update Storm's topology.classpath in Ambari to include '/etc/hbase/conf:/etc/hadoop/conf'. Restart Storm service.
+
+Place the mpack's tar.gz onto the node running Ambari Server. From the command line on this node, run
+```
+ambari-server install-mpack --mpack=<mpack_location> --verbose
+```
+
+This will make the services available in Ambari in the same manner as any services in a stack, e.g. through Add Services or during cluster install.
+The Indexing / Parsers/ Enrichment masters should be colocated with a Kafka Broker (to create topics) and HBase client (to create the enrichment and theatintel tables).
+This colocation is currently not enforced by Ambari, and should be managed by either a Service or Stack advisor as an enhancement.
+
+Several configuration parameters will need to be filled in, and should be pretty self explanatory (primarily a couple of Elasticsearch configs, and the Storm REST URL). Examples are provided in the descriptions on Ambari.
+Notably, the URL for the GeoIP database that is preloaded (and is prefilled by default) can be set to use a `file://` location
+
+After installation, a custom action is available in Ambari (where stop / start services are) to install Elasticsearch templates. Similar to this, a custom Kibana action to Load Template is available.
+
+#### Offline installation
+Currently there is only one point that would reach out to the internet during an install. This is the URL for the GeoIP database information that is preloaded into MySQL.
+
+The RPMs DO NOT reach out to the internet (because there is currently no hosting for them). They look on the local filesystem in `/localrepo`.
+
+### Current Limitations
+There are a set of limitations that should be addressed based to improve the current state of the mpacks.
+
+- MySQL install should be optional (and allow for using an existing instance).
+- MySQL should not be installed on a node already running a MySQL instance (e.g. an Ambari Server using MySQL as its database).
+- There is currently no hosting for RPMs remotely. They will have to be built locally.
+- Colocation of appropriate services should be enforced by Ambari. See [#Installing Management Pack] for more details.
+- Storm's topology.classpath is not updated with the Metron service install and needs to be updated separately.
+- Several configuration parameters used when installing the Metron service could (and should) be grabbed from Ambari. Install will require them to be manually entered.
+- Need to handle upgrading Metron
+
## RPM
RPMs can be built to install the components in metron-platform. These RPMs are built in a Docker container and placed into `target`.
@@ -111,7 +169,7 @@ Components in the RPMs:
### Building RPMs
From `metron-deployment` run
```
-mvn clean package
+mvn clean package -Pbuild-rpms
```
The output RPM files will land in `target/RPMS/noarch`. They can be installed with the standard
@@ -122,3 +180,4 @@ rpm -i <package>
## TODO
- migrate existing MySQL/GeoLite playbook
- Support Ubuntu deployments
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
new file mode 100644
index 0000000..0a40f75
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -0,0 +1,111 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.metron.packaging.mpacks</groupId>
+ <artifactId>metron_mpack</artifactId>
+ <version>1.0.0.0-SNAPSHOT</version>
+ <name>Metron Ambari Management Pack</name>
+
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-deployment</artifactId>
+ <version>0.2.0BETA</version>
+ <relativePath>../../..</relativePath>
+ </parent>
+
+ <dependencies>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>parse-version</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>parse-version</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>regex-property</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>mpackVersion</name>
+ <value>${project.version}</value>
+ <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex>
+ <replacement>$1.$2.$3.$4</replacement>
+ <failIfNoMatch>false</failIfNoMatch>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>mpack.json</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptors>
+ <descriptor>src/main/assemblies/metron-mpack.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
new file mode 100644
index 0000000..7de489f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
@@ -0,0 +1,43 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+ <id></id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>src/main/resources/common-services</directory>
+ <outputDirectory>common-services</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources/addon-services</directory>
+ <outputDirectory>addon-services</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <files>
+ <file>
+ <source>target/mpack.json</source>
+ </file>
+ </files>
+ <dependencySets>
+ </dependencySets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml
new file mode 100755
index 0000000..e2e6cdd
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ELASTICSEARCH</name>
+ <version>2.3.3</version>
+ <extends>common-services/ELASTICSEARCH/2.3.3</extends>
+ </service>
+ </services>
+</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml
new file mode 100755
index 0000000..156a34a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ELASTICSEARCH</name>
+ <extends>common-services/ELASTICSEARCH/2.3.3</extends>
+ </service>
+ <service>
+ <name>KIBANA</name>
+ <extends>common-services/KIBANA/4.5.1</extends>
+ </service>
+ </services>
+</metainfo>
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json
new file mode 100755
index 0000000..622a512
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json
@@ -0,0 +1,27 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "kibana_master_ui",
+ "label": "Metron Dashboard",
+ "requires_user_name": "false",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "kibana_server_port",
+ "http_default_port": "5601",
+ "https_property": "kibana_server_port",
+ "https_default_port": "5601",
+ "regex": "^(\\d+)$",
+ "site": "kibana-env"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml
new file mode 100755
index 0000000..a3c984c
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>METRON</name>
+ <version>0.2.0BETA</version>
+ <extends>common-services/METRON/0.2.0BETA</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json
new file mode 100755
index 0000000..0f4cd39
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json
@@ -0,0 +1,14 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
+ "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
+ "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
+ "METRON_ENRICHMENT_MYSQL_SERVER-STOP" : ["METRON_ENRICHMENT_MASTER-STOP"],
+ "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
+ "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
+ "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
new file mode 100755
index 0000000..2939c28
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>elastic_user</name>
+ <value>elasticsearch</value>
+ <property-type>USER</property-type>
+ <description>The user for Elasticsearch</description>
+ </property>
+ <property>
+ <name>user_group</name>
+ <value>elasticsearch</value>
+ <description>The group for Elasticsearch</description>
+ </property>
+ <property>
+ <name>elastic_log_dir</name>
+ <value>/var/log/elasticsearch</value>
+ <description>Log directory for elastic</description>
+ </property>
+ <property>
+ <name>elastic_pid_dir</name>
+ <value>/var/run/elasticsearch</value>
+ <description>The directory for pid files</description>
+ </property>
+
+ <!-- elasticsearch-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for elastic-env.sh file</description>
+ <value>
+#!/bin/bash
+
+# Set ELASTICSEARCH specific environment variables here.
+
+# The java implementation to use.
+export JAVA_HOME={{java64_home}}
+export PATH=$PATH:$JAVA_HOME/bin
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
new file mode 100755
index 0000000..fb3a443
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
@@ -0,0 +1,180 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Elastic search Configurations -->
+
+<configuration supports_final="true">
+ <!-- Configurations -->
+ <property>
+ <name>cluster_name</name>
+ <value>metron</value>
+ <description>Cluster name identifies your cluster</description>
+ </property>
+ <property>
+ <name>zen_discovery_ping_unicast_hosts</name>
+ <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.-->
+ <value></value>
+ <description>Unicast discovery list of hosts to act as gossip routers, in comma separated format.</description>
+ </property>
+ <property>
+ <name>index_number_of_shards</name>
+ <value>4</value>
+ <description>Set the number of shards (splits) of an index</description>
+ </property>
+ <property>
+ <name>index_number_of_replicas</name>
+ <value>2</value>
+ <description>Set the number of replicas (additional copies) of an index</description>
+ </property>
+ <!-- Logging Configurations -->
+ <property>
+ <name>path_data</name>
+ <value>"/opt/lmm/es_data"</value>
+ <description>Path to directory where to store index data allocated for this node. e.g. "/mnt/first", "/mnt/second"</description>
+ </property>
+ <!-- Discovery -->
+ <property>
+ <name>transport_tcp_port</name>
+ <value>9300-9400</value>
+ <description>Set a custom port for the node to node communication</description>
+ </property>
+ <property>
+ <name>http_port</name>
+ <value>9200-9300</value>
+ <description>Set a custom port to listen for HTTP traffic</description>
+ </property>
+ <!-- Discovery -->
+ <property>
+ <name>discovery_zen_ping_multicast_enabled</name>
+ <value>false</value>
+ <description>master eligible nodes</description>
+ </property>
+ <property>
+ <name>discovery_zen_ping_timeout</name>
+ <value>3s</value>
+ <description>Wait for ping responses for master discovery</description>
+ </property>
+ <property>
+ <name>discovery_zen_fd_ping_interval</name>
+ <value>15s</value>
+ <description>Wait for ping for cluster discovery</description>
+ </property>
+ <property>
+ <name>discovery_zen_fd_ping_timeout</name>
+ <value>60s</value>
+ <description>Wait for ping for cluster discovery</description>
+ </property>
+ <property>
+ <name>discovery_zen_fd_ping_retries</name>
+ <value>5</value>
+ <description>Number of ping retries before blacklisting</description>
+ </property>
+ <!-- Gateway -->
+ <property>
+ <name>gateway_recover_after_data_nodes</name>
+ <value>3</value>
+ <description>Recover as long as this many data or master nodes have joined the cluster.</description>
+ </property>
+ <property>
+ <name>recover_after_time</name>
+ <value>15m</value>
+ <description>recover_after_time</description>
+ </property>
+ <property>
+ <name>expected_data_nodes</name>
+ <value>0</value>
+ <description>expected_data_nodes</description>
+ </property>
+ <!-- Index -->
+ <property>
+ <name>index_merge_scheduler_max_thread_count</name>
+ <value>5</value>
+ <description>index.merge.scheduler.max_thread_count</description>
+ </property>
+ <property>
+ <name>indices_memory_index_store_throttle_type</name>
+ <value>none</value>
+ <description>index_store_throttle_type</description>
+ </property>
+ <property>
+ <name>index_refresh_interval</name>
+ <value>1s</value>
+ <description>index refresh interval</description>
+ </property>
+ <property>
+ <name>index_translog_flush_threshold_size</name>
+ <value>5g</value>
+ <description>index_translog_flush_threshold_size</description>
+ </property>
+ <property>
+ <name>indices_memory_index_buffer_size</name>
+ <value>10%</value>
+ <description>Percentage of heap used for write buffers</description>
+ </property>
+ <property>
+ <name>bootstrap_mlockall</name>
+ <value>true</value>
+ <description>The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out</description>
+ </property>
+ <property>
+ <name>threadpool_bulk_queue_size</name>
+ <value>3000</value>
+ <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request</description>
+ </property>
+ <property>
+ <name>threadpool_index_queue_size</name>
+ <value>1000</value>
+ <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request</description>
+ </property>
+ <property>
+ <name>indices_cluster_send_refresh_mapping</name>
+ <value>false</value>
+ <description>In order to make the index request more efficient, we have set this property on our data nodes</description>
+ </property>
+ <property>
+ <name>indices_fielddata_cache_size</name>
+ <value>25%</value>
+ <description>You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_disk_watermark_high</name>
+ <value>0.99</value>
+ <description>Property used when multiple drives are used to understand max thresholds</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_disk_threshold_enabled</name>
+ <value>true</value>
+ <description>Property used when multiple drives are used to understand if thresholding is active</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_disk_watermark_low</name>
+ <value>.97</value>
+ <description>Property used when multiple drives are used to understand min thresholds</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_node_concurrent_recoveries</name>
+ <value>4</value>
+ <description>Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart</description>
+ </property>
+ <property>
+ <name>network_host</name>
+ <value>_lo_,_eth0_</value>
+ <description>Network interface(s) will bind to. </description>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
new file mode 100755
index 0000000..58e4916
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
@@ -0,0 +1,98 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>elastic_home</name>
+ <value>/usr/share/elasticsearch/</value>
+ <description>Elasticsearch Home Directory</description>
+ </property>
+ <property>
+ <name>data_dir</name>
+ <value>/var/lib/elasticsearch/</value>
+ <description>Elasticsearch Data Directory</description>
+ </property>
+ <property>
+ <name>work_dir</name>
+ <value>/tmp/elasticsearch/</value>
+ <description>Elasticsearch Work Directory</description>
+ </property>
+ <property>
+ <name>conf_dir</name>
+ <value>/etc/elasticsearch/</value>
+ <description>Elasticsearch Configuration Directory</description>
+ </property>
+ <property>
+ <name>heap_size</name>
+ <value>128m</value>
+ <description>Heap size</description>
+ </property>
+ <property>
+ <name>max_open_files</name>
+ <value>65535</value>
+ <description>Maximum number of open files</description>
+ </property>
+ <property>
+ <name>max_map_count</name>
+ <value>262144</value>
+ <description>Maximum number of memory map areas for process</description>
+ </property>
+
+ <!--/etc/sysconfig/elasticsearch-->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for elastic-env.sh file</description>
+ <value>
+# Directory where the Elasticsearch binary distribution resides
+ES_HOME={{elastic_home}}
+
+# Heap Size (defaults to 256m min, 1g max)
+ES_HEAP_SIZE={{heap_size}}
+
+# Maximum number of open files
+MAX_OPEN_FILES={{max_open_files}}
+
+# Maximum number of VMA (Virtual Memory Areas) a process can own
+MAX_MAP_COUNT={{max_map_count}}
+
+# Elasticsearch log directory
+LOG_DIR={{log_dir}}
+
+# Elasticsearch data directory
+DATA_DIR={{data_dir}}
+
+# Elasticsearch work directory
+WORK_DIR={{work_dir}}
+
+# Elasticsearch conf directory
+CONF_DIR={{conf_dir}}
+
+# User to run as, change this to a specific elasticsearch user if possible
+# Also make sure, this user can write into the log directories in case you change them
+# This setting only works for the init script, but has to be configured separately for systemd startup
+ES_USER={{elastic_user}}
+
+# Additional Java OPTS
+ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled
+-XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
+-XX:ErrorFile={{log_dir}}elasticsearch_err.log -XX:ParallelGCThreads=8"
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
new file mode 100755
index 0000000..25f98e5
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ELASTICSEARCH</name>
+ <displayName>Elasticsearch</displayName>
+ <comment>Indexing and Search</comment>
+ <version>2.3.3</version>
+ <components>
+ <component>
+ <name>ES_MASTER</name>
+ <displayName>Elasticsearch Master</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/elastic_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ <component>
+ <name>ES_SLAVE</name>
+ <displayName>Elasticsearch Data Node</displayName>
+ <category>SLAVE</category>
+ <cardinality>3+</cardinality>
+ <commandScript>
+ <script>scripts/elastic_slave.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>elasticsearch-2.3.3</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+ <configuration-dependencies>
+ <config-type>elastic-env</config-type>
+ <config-type>elastic-site</config-type>
+ <config-type>elastic-sysconfig</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ </service>
+ </services>
+</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
new file mode 100755
index 0000000..a472655
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+
+
+def elastic():
+ print "INSIDE THE %s" % __file__
+ import params
+
+ params.path_data = params.path_data.replace('"', '')
+ data_path = params.path_data.replace(' ', '').split(',')
+ data_path[:] = [x.replace('"', '') for x in data_path]
+
+ directories = [params.log_dir, params.pid_dir, params.conf_dir]
+ directories = directories + data_path
+
+ Directory(directories,
+ create_parents=True,
+ # recursive=True,
+ mode=0755,
+ owner=params.elastic_user,
+ group=params.elastic_user
+ )
+
+ print "Master env: ""{0}/elastic-env.sh".format(params.conf_dir)
+ File("{0}/elastic-env.sh".format(params.conf_dir),
+ owner=params.elastic_user,
+ content=InlineTemplate(params.elastic_env_sh_template)
+ )
+
+ configurations = params.config['configurations']['elastic-site']
+
+ print "Master yml: ""{0}/elasticsearch.yml".format(params.conf_dir)
+ File("{0}/elasticsearch.yml".format(params.conf_dir),
+ content=Template(
+ "elasticsearch.master.yaml.j2",
+ configurations=configurations),
+ owner=params.elastic_user,
+ group=params.elastic_user
+ )
+
+ print "Master sysconfig: /etc/sysconfig/elasticsearch"
+ File(format("/etc/sysconfig/elasticsearch"),
+ owner="root",
+ group="root",
+ content=InlineTemplate(params.sysconfig_template)
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
new file mode 100755
index 0000000..3ad4e1f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
@@ -0,0 +1,81 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.script import Script
+
+from elastic import elastic
+
+
+class Elasticsearch(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+
+ print 'Install the Master'
+ Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
+ Execute("echo \"[elasticsearch-2.x]\n"
+ "name=Elasticsearch repository for 2.x packages\n"
+ "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n"
+ "gpgcheck=1\n"
+ "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
+ "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo")
+
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+
+ elastic()
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ stop_cmd = format("service elasticsearch stop")
+ print 'Stop the Master'
+ Execute(stop_cmd)
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ self.configure(env)
+ start_cmd = format("service elasticsearch start")
+ print 'Start the Master'
+ Execute(start_cmd)
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ status_cmd = format("service elasticsearch status")
+ print 'Status of the Master'
+ Execute(status_cmd)
+
+ def restart(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ restart_cmd = format("service elasticsearch restart")
+ print 'Restarting the Master'
+ Execute(restart_cmd)
+
+
+if __name__ == "__main__":
+ Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
new file mode 100755
index 0000000..4b636e8
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
@@ -0,0 +1,77 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.script import Script
+
+from slave import slave
+
+
+class Elasticsearch(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+ print 'Install the Slave'
+ Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
+ Execute("echo \"[elasticsearch-2.x]\n"
+ "name=Elasticsearch repository for 2.x packages\n"
+ "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n"
+ "gpgcheck=1\n"
+ "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
+ "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo")
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+ slave()
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ stop_cmd = format("service elasticsearch stop")
+ print 'Stop the Slave'
+ Execute(stop_cmd)
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ start_cmd = format("service elasticsearch start")
+ print 'Start the Slave'
+ Execute(start_cmd)
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ status_cmd = format("service elasticsearch status")
+ print 'Status of the Slave'
+ Execute(status_cmd)
+
+ def restart(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ restart_cmd = format("service elasticsearch restart")
+ print 'Restarting the Slave'
+ Execute(restart_cmd)
+
+
+if __name__ == "__main__":
+ Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
new file mode 100755
index 0000000..30e88ea
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+
+# server configurations
+config = Script.get_config()
+
+elastic_home = config['configurations']['elastic-sysconfig']['elastic_home']
+data_dir = config['configurations']['elastic-sysconfig']['data_dir']
+work_dir = config['configurations']['elastic-sysconfig']['work_dir']
+conf_dir = config['configurations']['elastic-sysconfig']['conf_dir']
+heap_size = config['configurations']['elastic-sysconfig']['heap_size']
+max_open_files = config['configurations']['elastic-sysconfig']['max_open_files']
+max_map_count = config['configurations']['elastic-sysconfig']['max_map_count']
+
+elastic_user = config['configurations']['elastic-env']['elastic_user']
+user_group = config['configurations']['elastic-env']['user_group']
+log_dir = config['configurations']['elastic-env']['elastic_log_dir']
+pid_dir = '/var/run/elasticsearch'
+pid_file = '/var/run/elasticsearch/elasticsearch.pid'
+hostname = config['hostname']
+java64_home = config['hostLevelParams']['java_home']
+elastic_env_sh_template = config['configurations']['elastic-env']['content']
+sysconfig_template = config['configurations']['elastic-sysconfig']['content']
+
+cluster_name = config['configurations']['elastic-site']['cluster_name']
+zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts']
+
+path_data = config['configurations']['elastic-site']['path_data']
+http_port = config['configurations']['elastic-site']['http_port']
+transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port']
+
+recover_after_time = config['configurations']['elastic-site']['recover_after_time']
+gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes']
+expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes']
+discovery_zen_ping_multicast_enabled = config['configurations']['elastic-site']['discovery_zen_ping_multicast_enabled']
+index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count']
+index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size']
+index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval']
+indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type']
+index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards']
+index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas']
+indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size']
+bootstrap_mlockall = config['configurations']['elastic-site']['bootstrap_mlockall']
+threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size']
+cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries']
+cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low']
+cluster_routing_allocation_disk_threshold_enabled = config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled']
+cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high']
+indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size']
+indices_cluster_send_refresh_mapping = config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping']
+threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size']
+
+discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout']
+discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval']
+discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout']
+discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries']
+
+network_host = config['configurations']['elastic-site']['network_host']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
new file mode 100755
index 0000000..ef9f6dd
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+
+
+def properties_inline_template(configurations):
+ return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }}
+{% endfor %}''', configurations_dict=configurations)
+
+
+def properties_config(filename, configurations=None, conf_dir=None,
+ mode=None, owner=None, group=None, brokerid=None):
+ config_content = properties_inline_template(configurations)
+ File(format("{conf_dir}/{filename}"), content=config_content, owner=owner,
+ group=group, mode=mode)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
new file mode 100755
index 0000000..e84fb01
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+import subprocess
+import sys
+
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.script import Script
+
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ doc = '{"name": "Ambari Smoke test"}'
+ index = "ambari_smoke_test"
+
+ print("Running Elastic search service check", file=sys.stdout)
+
+ # Make sure the service is actually up. We can live without everything allocated.
+ # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can
+ # also hit ES before things are green.
+ host = "localhost:9200"
+ Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host,
+ logoutput=True,
+ tries=6,
+ try_sleep=20
+ )
+
+ # Put a document into a new index.
+
+ Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True)
+
+ # Retrieve the document. Use subprocess because we actually need the results here.
+ cmd_retrieve = "curl -XGET '%s/%s/test/1'" % (host, index)
+ proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+ response_retrieve = stdout
+ print("Retrieval response is: %s" % response_retrieve)
+ expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \
+ % (index, doc)
+
+ # Delete the index
+ cmd_delete = "curl -XDELETE '%s/%s'" % (host, index)
+ proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+ response_delete = stdout
+ print("Delete index response is: %s" % response_retrieve)
+ expected_delete = '{"acknowledged":true}'
+
+ if (expected_retrieve == response_retrieve) and (expected_delete == response_delete):
+ print("Smoke test able to communicate with Elasticsearch")
+ else:
+ print("Elasticsearch service unable to retrieve document.")
+ sys.exit(1)
+
+ exit(0)
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
new file mode 100755
index 0000000..90d4d3d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+
+
+def slave():
+ import params
+
+ params.path_data = params.path_data.replace('"', '')
+ data_path = params.path_data.replace(' ', '').split(',')
+ data_path[:] = [x.replace('"', '') for x in data_path]
+
+ directories = [params.log_dir, params.pid_dir, params.conf_dir]
+ directories = directories + data_path
+
+ Directory(directories,
+ create_parents=True,
+ mode=0755,
+ owner=params.elastic_user,
+ group=params.elastic_user,
+ cd_access="a"
+ )
+
+ File("{0}/elastic-env.sh".format(params.conf_dir),
+ owner=params.elastic_user,
+ content=InlineTemplate(params.elastic_env_sh_template)
+ )
+
+ configurations = params.config['configurations']['elastic-site']
+
+ File("{0}/elasticsearch.yml".format(params.conf_dir),
+ content=Template(
+ "elasticsearch.slave.yaml.j2",
+ configurations=configurations),
+ owner=params.elastic_user,
+ group=params.elastic_user
+ )
+
+ print "Master sysconfig: /etc/sysconfig/elasticsearch"
+ File(format("/etc/sysconfig/elasticsearch"),
+ owner="root",
+ group="root",
+ content=InlineTemplate(params.sysconfig_template)
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
new file mode 100755
index 0000000..9cfb5cf
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+
+config = Script.get_config()
+
+elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
+elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid")
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
new file mode 100755
index 0000000..a9de018
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
@@ -0,0 +1,84 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+cluster:
+ name: {{cluster_name}}
+ routing:
+ allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
+ allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
+ allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
+ allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
+
+discovery:
+ zen:
+ ping:
+ multicast:
+ enabled: {{discovery_zen_ping_multicast_enabled}}
+ unicast:
+ hosts: "{{zen_discovery_ping_unicast_hosts}}"
+
+node:
+ data: false
+ master: true
+ name: {{hostname}}
+path:
+ data: {{path_data}}
+
+http.cors.enabled: true
+
+port: {{http_port}}
+
+transport:
+ tcp:
+ port: {{transport_tcp_port}}
+
+gateway:
+ recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
+ recover_after_time: {{recover_after_time}}
+ expected_data_nodes: {{expected_data_nodes}}
+
+index:
+ number_of_shards: {{index_number_of_shards}}
+ merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}}
+ translog.flush_threshold_size: {{index_translog_flush_threshold_size}}
+ refresh_interval: {{index_refresh_interval}}
+ number_of_replicas: {{index_number_of_replicas}}
+
+indices:
+ memory:
+ index_buffer_size: {{indices_memory_index_buffer_size}}
+ store.throttle.type: {{indices_memory_index_store_throttle_type}}
+ fielddata:
+ cache.size: {{indices_fielddata_cache_size}}
+ cluster:
+ send_refresh_mapping: {{indices_cluster_send_refresh_mapping}}
+
+bootstrap.mlockall: {{bootstrap_mlockall}}
+
+threadpool:
+ bulk:
+ queue_size: {{threadpool_bulk_queue_size}}
+ index:
+ queue_size: {{threadpool_index_queue_size}}
+
+discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
+discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
+discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
+discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
+
+network.host: {{network_host}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
new file mode 100755
index 0000000..e88fc5f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
@@ -0,0 +1,84 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+cluster:
+ name: {{cluster_name}}
+ routing:
+ allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
+ allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
+ allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
+ allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
+
+discovery:
+ zen:
+ ping:
+ multicast:
+ enabled: {{discovery_zen_ping_multicast_enabled}}
+ unicast:
+ hosts: "{{zen_discovery_ping_unicast_hosts}}"
+
+node:
+ data: true
+ master: false
+ name: {{hostname}}
+path:
+ data: {{path_data}}
+
+http.cors.enabled: true
+
+port: {{http_port}}
+
+transport:
+ tcp:
+ port: {{transport_tcp_port}}
+
+gateway:
+ recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
+ recover_after_time: {{recover_after_time}}
+ expected_data_nodes: {{expected_data_nodes}}
+
+index:
+ number_of_shards: {{index_number_of_shards}}
+ merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}}
+ translog.flush_threshold_size: {{index_translog_flush_threshold_size}}
+ refresh_interval: {{index_refresh_interval}}
+ number_of_replicas: {{index_number_of_replicas}}
+
+indices:
+ memory:
+ index_buffer_size: {{indices_memory_index_buffer_size}}
+ store.throttle.type: {{indices_memory_index_store_throttle_type}}
+ fielddata:
+ cache.size: {{indices_fielddata_cache_size}}
+ cluster:
+ send_refresh_mapping: {{indices_cluster_send_refresh_mapping}}
+
+bootstrap.mlockall: {{bootstrap_mlockall}}
+
+threadpool:
+ bulk:
+ queue_size: {{threadpool_bulk_queue_size}}
+ index:
+ queue_size: {{threadpool_index_queue_size}}
+
+discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
+discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
+discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
+discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
+
+network.host: {{network_host}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
new file mode 100755
index 0000000..130d018
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
@@ -0,0 +1,8 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
new file mode 100755
index 0000000..972945d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>kibana_user</name>
+ <value>kibana</value>
+ <property-type>USER</property-type>
+ <description></description>
+ </property>
+ <property>
+ <name>user_group</name>
+ <value>kibana</value>
+ <property-type>GROUP</property-type>
+ <description></description>
+ </property>
+ <property require-input="true">
+ <name>kibana_log_dir</name>
+ <value>/var/log/kibana</value>
+ <description>Log directory for Kibana</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_pid_dir</name>
+ <value>/var/run/kibana</value>
+ <description>PID directory for Kibana</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_es_url</name>
+ <value></value>
+ <description>The Elasticsearch instance to use for all your queries. (http://eshost:9200)</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_server_port</name>
+ <value>5000</value>
+ <description>Kibana back end server port to use.</description>
+ </property>
+</configuration>