You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2016/12/13 18:25:46 UTC
[4/4] incubator-metron git commit: METRON-580: Remove hard-coded
Metron version from Ambari MPack code (mmiklavc) closes
apache/incubator-metron#364
METRON-580: Remove hard-coded Metron version from Ambari MPack code (mmiklavc) closes apache/incubator-metron#364
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/64a49ada
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/64a49ada
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/64a49ada
Branch: refs/heads/master
Commit: 64a49adad0df51c9b87422be8edd9cf1fff90dac
Parents: 506610c
Author: mmiklavc <mi...@gmail.com>
Authored: Tue Dec 13 11:25:00 2016 -0700
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Tue Dec 13 11:25:00 2016 -0700
----------------------------------------------------------------------
metron-analytics/metron-profiler/README.md | 4 +-
.../packaging/ambari/metron-mpack/pom.xml | 23 +-
.../src/main/assemblies/metron-mpack.xml | 16 +
.../addon-services/METRON/0.3.0/metainfo.xml | 26 --
.../METRON/0.3.0/role_command_order.json | 14 -
.../addon-services/METRON/CURRENT/metainfo.xml | 26 ++
.../METRON/CURRENT/role_command_order.json | 14 +
.../METRON/0.3.0/configuration/metron-env.xml | 262 ----------------
.../common-services/METRON/0.3.0/metainfo.xml | 295 -------------------
.../METRON/0.3.0/package/files/addMysqlUser.sh | 51 ----
.../0.3.0/package/files/bro_index.template | 217 --------------
.../0.3.0/package/files/createMysqlGeoIp.sh | 53 ----
.../0.3.0/package/files/removeMysqlUser.sh | 43 ---
.../0.3.0/package/files/snort_index.template | 183 ------------
.../0.3.0/package/files/yaf_index.template | 205 -------------
.../package/scripts/enrichment_commands.py | 174 -----------
.../0.3.0/package/scripts/enrichment_master.py | 86 ------
.../0.3.0/package/scripts/indexing_commands.py | 154 ----------
.../0.3.0/package/scripts/indexing_master.py | 120 --------
.../0.3.0/package/scripts/metron_service.py | 76 -----
.../0.3.0/package/scripts/mysql_server.py | 63 ----
.../0.3.0/package/scripts/mysql_service.py | 46 ---
.../METRON/0.3.0/package/scripts/mysql_users.py | 58 ----
.../METRON/0.3.0/package/scripts/mysql_utils.py | 56 ----
.../0.3.0/package/scripts/params/__init__.py | 18 --
.../0.3.0/package/scripts/params/params.py | 32 --
.../package/scripts/params/params_linux.py | 190 ------------
.../package/scripts/params/params_windows.py | 20 --
.../package/scripts/params/status_params.py | 59 ----
.../0.3.0/package/scripts/parser_commands.py | 208 -------------
.../0.3.0/package/scripts/parser_master.py | 88 ------
.../0.3.0/package/scripts/service_check.py | 41 ---
.../package/templates/enrichment.properties.j2 | 90 ------
.../METRON/0.3.0/quicklinks/quicklinks.json | 28 --
.../METRON/0.3.0/service_advisor.py | 159 ----------
.../METRON/0.3.0/themes/metron_theme.json | 219 --------------
.../METRON/CURRENT/configuration/metron-env.xml | 262 ++++++++++++++++
.../common-services/METRON/CURRENT/metainfo.xml | 295 +++++++++++++++++++
.../CURRENT/package/files/addMysqlUser.sh | 51 ++++
.../CURRENT/package/files/bro_index.template | 217 ++++++++++++++
.../CURRENT/package/files/createMysqlGeoIp.sh | 53 ++++
.../CURRENT/package/files/removeMysqlUser.sh | 43 +++
.../CURRENT/package/files/snort_index.template | 183 ++++++++++++
.../CURRENT/package/files/yaf_index.template | 205 +++++++++++++
.../package/scripts/enrichment_commands.py | 174 +++++++++++
.../package/scripts/enrichment_master.py | 86 ++++++
.../package/scripts/indexing_commands.py | 154 ++++++++++
.../CURRENT/package/scripts/indexing_master.py | 120 ++++++++
.../CURRENT/package/scripts/metron_service.py | 76 +++++
.../CURRENT/package/scripts/mysql_server.py | 63 ++++
.../CURRENT/package/scripts/mysql_service.py | 46 +++
.../CURRENT/package/scripts/mysql_users.py | 58 ++++
.../CURRENT/package/scripts/mysql_utils.py | 56 ++++
.../CURRENT/package/scripts/params/__init__.py | 18 ++
.../CURRENT/package/scripts/params/params.py | 32 ++
.../package/scripts/params/params_linux.py | 190 ++++++++++++
.../package/scripts/params/params_windows.py | 20 ++
.../package/scripts/params/status_params.py | 59 ++++
.../CURRENT/package/scripts/parser_commands.py | 208 +++++++++++++
.../CURRENT/package/scripts/parser_master.py | 88 ++++++
.../CURRENT/package/scripts/service_check.py | 41 +++
.../package/templates/enrichment.properties.j2 | 90 ++++++
.../METRON/CURRENT/quicklinks/quicklinks.json | 28 ++
.../METRON/CURRENT/service_advisor.py | 159 ++++++++++
.../METRON/CURRENT/themes/metron_theme.json | 219 ++++++++++++++
.../metron-mpack/src/main/resources/mpack.json | 2 +-
.../roles/sensor-test-mode/tasks/yaf.yml | 1 +
67 files changed, 3372 insertions(+), 3342 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-analytics/metron-profiler/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler/README.md
index 6a853a1..2d347d2 100644
--- a/metron-analytics/metron-profiler/README.md
+++ b/metron-analytics/metron-profiler/README.md
@@ -41,7 +41,7 @@ This section will describe the steps required to get your first profile running.
1. Upload the profile definition to Zookeeper.
```
- $ cd /usr/metron/0.3.0/
+ $ cd $METRON_HOME
$ bin/zk_load_configs.sh -m PUSH -i config/zookeeper/ -z node1:2181
```
@@ -174,7 +174,7 @@ The Profiler runs as an independent Storm topology. The configuration for the P
After altering the configuration, start the Profiler.
```
-$ /usr/metron/0.3.0/start_profiler_topology.sh
+$ $METRON_HOME/start_profiler_topology.sh
```
## Examples
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 2eb4775..6f3f7a0 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -48,15 +48,28 @@
</goals>
</execution>
<execution>
- <id>regex-property</id>
+ <id>metron-version</id>
<goals>
<goal>regex-property</goal>
</goals>
<configuration>
- <name>mpackVersion</name>
- <value>${project.version}</value>
- <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex>
- <replacement>$1.$2.$3.$4</replacement>
+ <name>metron.version</name>
+ <value>${project.parent.version}</value>
+ <regex>-SNAPSHOT</regex>
+ <replacement></replacement>
+ <failIfNoMatch>false</failIfNoMatch>
+ </configuration>
+ </execution>
+ <execution>
+ <id>metron-short-version</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>metron.short.version</name>
+ <value>${project.parent.version}</value>
+ <regex>\.|-SNAPSHOT</regex>
+ <replacement></replacement>
<failIfNoMatch>false</failIfNoMatch>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
index 238b028..ec4272e 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
@@ -25,12 +25,28 @@
<includeBaseDirectory>true</includeBaseDirectory>
<fileSets>
<fileSet>
+ <directory>src/main/resources/common-services/METRON/CURRENT</directory>
+ <outputDirectory>common-services/METRON/${metron.version}</outputDirectory>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
<directory>src/main/resources/common-services</directory>
<outputDirectory>common-services</outputDirectory>
+ <excludes>
+ <exclude>**/CURRENT/**</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources/addon-services/METRON/CURRENT</directory>
+ <outputDirectory>addon-services/METRON/${metron.version}</outputDirectory>
+ <filtered>true</filtered>
</fileSet>
<fileSet>
<directory>src/main/resources/addon-services</directory>
<outputDirectory>addon-services</outputDirectory>
+ <excludes>
+ <exclude>**/CURRENT/**</exclude>
+ </excludes>
</fileSet>
</fileSets>
<files>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/metainfo.xml
deleted file mode 100755
index 86231ec..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/metainfo.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>METRON</name>
- <version>0.3.0</version>
- <extends>common-services/METRON/0.3.0</extends>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/role_command_order.json
deleted file mode 100755
index 0f4cd39..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/role_command_order.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "_comment" : "Record format:",
- "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps" : {
- "_comment" : "dependencies for all cases",
- "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
- "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
- "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
- "METRON_ENRICHMENT_MYSQL_SERVER-STOP" : ["METRON_ENRICHMENT_MASTER-STOP"],
- "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
- "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
- "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/metainfo.xml
new file mode 100755
index 0000000..3edff64
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>METRON</name>
+ <version>${metron.version}</version>
+ <extends>common-services/METRON/${metron.version}</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
new file mode 100755
index 0000000..0f4cd39
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
@@ -0,0 +1,14 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
+ "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
+ "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
+ "METRON_ENRICHMENT_MYSQL_SERVER-STOP" : ["METRON_ENRICHMENT_MASTER-STOP"],
+ "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
+ "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
+ "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/configuration/metron-env.xml
deleted file mode 100644
index ec6ef4c..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/configuration/metron-env.xml
+++ /dev/null
@@ -1,262 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration supports_final="true">
- <property>
- <name>metron_home</name>
- <value>/usr/metron/0.3.0</value>
- <description>Metron home directory</description>
- <display-name>Metron home</display-name>
- </property>
- <property>
- <name>metron_apps_hdfs_dir</name>
- <value>/apps/metron</value>
- <description>Metron apps HDFS dir</description>
- <display-name>Metron apps HDFS dir</display-name>
- </property>
- <property>
- <name>metron_apps_indexed_hdfs_dir</name>
- <value>{{metron_apps_hdfs_dir}}/indexing/indexed</value>
- <description>Indexing bolts will write to this HDFS directory</description>
- <display-name>Metron apps indexed HDFS dir</display-name>
- </property>
- <property>
- <name>metron_zookeeper_config_dir</name>
- <value>config/zookeeper</value>
- <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
- <display-name>Metron Zookeeper config dir</display-name>
- </property>
- <property>
- <name>metron_user</name>
- <value>metron</value>
- <property-type>USER</property-type>
- <description>The user for Metron</description>
- <display-name>Metron User</display-name>
- </property>
- <property>
- <name>metron_group</name>
- <value>metron</value>
- <property-type>GROUP</property-type>
- <description>The group for Metron</description>
- <display-name>Metron Group Name</display-name>
- </property>
- <property>
- <name>metron_topic_retention</name>
- <description>Kafka Retention in GB</description>
- <value>10</value>
- <display-name>Topic Retention</display-name>
- </property>
- <property>
- <name>parsers</name>
- <value>bro,snort,yaf</value>
- <description>Metron parsers to deploy</description>
- <display-name>Metron Parsers</display-name>
- </property>
- <property>
- <name>metron_enrichment_db_user</name>
- <value>metron</value>
- <description>Database username to use to connect to the database.</description>
- <display-name>Enrichment Database User</display-name>
- </property>
- <property>
- <name>metron_enrichment_db_port</name>
- <display-name>Metron Enrichment Database Port</display-name>
- <value>3306</value>
- <description>Database port to use to connect to the database.</description>
- </property>
- <property>
- <name>metron_enrichment_db_password</name>
- <value></value>
- <property-type>PASSWORD</property-type>
- <display-name>Metron Enrichment Database Password</display-name>
- <description>Password to use against database</description>
- <value-attributes>
- <type>password</type>
- <overridable>false</overridable>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>mysql_admin_password</name>
- <value></value>
- <property-type>PASSWORD</property-type>
- <display-name>MySQL root user password</display-name>
- <description>Password to use to add Metron user to MySQL</description>
- <value-attributes>
- <type>password</type>
- <overridable>false</overridable>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>metron_indexing_topology</name>
- <value>indexing</value>
- <description>The Storm topology name for Indexing</description>
- <display-name>Indexing Topology Name</display-name>
- </property>
- <property>
- <name>es_cluster_name</name>
- <value>metron</value>
- <description>Name of Elasticsearch Cluster</description>
- <display-name>Elasticsearch Cluster Name</display-name>
- </property>
- <property>
- <name>geoip_url</name>
- <value>http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz</value>
- <description>Location of the GeoIP data to load.</description>
- <display-name>GEOIP Load Datafile URL</display-name>
- </property>
- <property require-input="true">
- <name>es_hosts</name>
- <value></value>
- <description>Comma delimited list of Elasticsearch Hosts. (eshost1,eshost2)</description>
- <display-name>Elasticsearch Hosts</display-name>
- </property>
- <property>
- <name>es_binary_port</name>
- <value>9300</value>
- <description>Elasticsearch binary port. (9300)</description>
- <display-name>Elasticsearch Binary Port</display-name>
- </property>
- <property>
- <name>es_http_port</name>
- <value>9200</value>
- <description>Elasticsearch HTTP port. (9200)</description>
- <display-name>Elasticsearch HTTP port</display-name>
- </property>
- <property require-input = "true">
- <name>storm_rest_addr</name>
- <display-name>Storm Rest Server Address</display-name>
- <description>URL of Storm UI (storm.ui.hostname:8744)</description>
- <value></value>
- </property>
- <property>
- <name>repo_type</name>
- <display-name>Repository Type</display-name>
- <description>Type of Repository: Local or Remote</description>
- <value>local</value>
- <value-attributes>
- <overridable>false</overridable>
- <type>value-list</type>
- <entries>
- <entry>
- <value>local</value>
- <label>Local</label>
- </entry>
- <entry>
- <value>remote</value>
- <label>Remote</label>
- </entry>
- </entries>
- <selection-cardinality>1</selection-cardinality>
- </value-attributes>
- </property>
- <property>
- <name>repo_url</name>
- <display-name>Repository URL</display-name>
- <value></value>
- </property>
- <property>
- <name>install_mysql</name>
- <description>Install New MySQL Instance for Enrichments</description>
- <display-name>Install MySQL</display-name>
- <value>Yes</value>
- <value-attributes>
- <overridable>false</overridable>
- <type>value-list</type>
- <entries>
- <entry>
- <value>Yes</value>
- <label>Yes</label>
- </entry>
- <entry>
- <value>No</value>
- <label>No</label>
- </entry>
- </entries>
- <selection-cardinality>1</selection-cardinality>
- </value-attributes>
- </property>
- <property>
- <name>mysql_host</name>
- <display-name>MySQL Host Address</display-name>
- <value></value>
- </property>
- <property>
- <name>global-json</name>
- <display-name>global.json template</display-name>
- <description>This is the jinja template for global.json file</description>
- <value>
-{
-"es.clustername": "{{ es_cluster_name }}",
-"es.ip": "{{ es_url }}",
-"es.date.format": "yyyy.MM.dd.HH"
-}
- </value>
- <value-attributes>
- <type>content</type>
- </value-attributes>
- </property>
- <property>
- <name>elasticsearch-properties</name>
- <description>The template for the elasticsearch.properties file.</description>
- <display-name>elasticsearch.properties template</display-name>
- <value>
-##### Storm #####
-indexing.workers=1
-indexing.executors=0
-##### Kafka #####
-kafka.zk={{ zookeeper_quorum }}
-kafka.broker={{ kafka_brokers }}
-kafka.start=WHERE_I_LEFT_OFF
-##### Indexing #####
-index.input.topic=indexing
-index.error.topic=indexing_error
-writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
-##### Metrics #####
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-#Graphite Addresses
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-##### HDFS #####
-bolt.hdfs.batch.size=5000
-bolt.hdfs.field.delimiter=|
-bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
-bolt.hdfs.rotation.policy.units=DAYS
-bolt.hdfs.rotation.policy.count=1
-bolt.hdfs.file.rotation.size.in.mb=5
-bolt.hdfs.file.system.url={{ default_fs }}
-bolt.hdfs.wip.file.path=/paloalto/wip
-bolt.hdfs.finished.file.path=/paloalto/rotated
-bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
-index.hdfs.output={{ metron_apps_indexed_hdfs_dir }}
- </value>
- <value-attributes>
- <type>content</type>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/metainfo.xml
deleted file mode 100644
index 5ae36b7..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/metainfo.xml
+++ /dev/null
@@ -1,295 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>METRON</name>
- <displayName>Metron</displayName>
- <comment>A scalable advanced security analytics framework built on Hadoop</comment>
- <version>0.3.0</version>
- <components>
-
- <component>
- <name>METRON_PARSERS</name>
- <displayName>Metron Parsers</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>false</versionAdvertised>
- <reassignAllowed>false</reassignAllowed>
- <clientsToUpdateConfigs></clientsToUpdateConfigs>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>STORM/SUPERVISOR</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/parser_master.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </component>
-
- <component>
- <name>METRON_ENRICHMENT_MASTER</name>
- <displayName>Metron Enrichment</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>false</versionAdvertised>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>HBASE/HBASE_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/enrichment_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
- <name>METRON_ENRICHMENT_MYSQL_SERVER</name>
- <displayName>MySQL Server</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>false</versionAdvertised>
- <clientsToUpdateConfigs></clientsToUpdateConfigs>
- <commandScript>
- <script>scripts/mysql_server.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>CLEAN</name>
- <commandScript>
- <script>scripts/mysql_server.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
- </component>
-
- <component>
- <name>METRON_INDEXING</name>
- <displayName>Metron Indexing</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>false</versionAdvertised>
- <reassignAllowed>false</reassignAllowed>
- <clientsToUpdateConfigs></clientsToUpdateConfigs>
- <dependencies>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>STORM/SUPERVISOR</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>ELASTICSEARCH_TEMPLATE_INSTALL</name>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- <customCommand>
- <name>ELASTICSEARCH_TEMPLATE_DELETE</name>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
- </component>
- </components>
-
- <themes>
- <theme>
- <fileName>metron_theme.json</fileName>
- <default>true</default>
- </theme>
- </themes>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>metron-common</name>
- </package>
- <package>
- <name>metron-data-management</name>
- </package>
- <package>
- <name>metron-parsers</name>
- </package>
- <package>
- <name>metron-enrichment</name>
- </package>
- <package>
- <name>metron-indexing</name>
- </package>
- <package>
- <name>metron-elasticsearch</name>
- </package>
- <package>
- <name>metron-pcap</name>
- </package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>redhat6,redhat7</osFamily>
- <packages>
- <package>
- <name>expect</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>redhat7</osFamily>
- <packages>
- <package>
- <name>mysql-community-release</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- <package>
- <name>mysql-community-server</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- <package>MySQL-python</package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>redhat6</osFamily>
- <packages>
- <package>
- <name>mysql-server</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- <package>
- <name>mysql</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>HDFS</service>
- <service>KAFKA</service>
- <service>STORM</service>
- <service>ZOOKEEPER</service>
- <service>HBASE</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>metron-env</config-type>
- <config-type>storm-env</config-type>
- <config-type>storm-site</config-type>
- <config-type>kafka-broker</config-type>
- <config-type>kafka-env</config-type>
- </configuration-dependencies>
- <restartRequiredAfterChange>true</restartRequiredAfterChange>
- <quickLinksConfigurations>
- <quickLinksConfiguration>
- <fileName>quicklinks.json</fileName>
- <default>true</default>
- </quickLinksConfiguration>
- </quickLinksConfigurations>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/addMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/addMysqlUser.sh
deleted file mode 100755
index 8a6a788..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/addMysqlUser.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-mysqldbuser=$1
-mysqldbpasswd=$2
-mysqldbhost=$3
-mysqlqdminpassword=$4
-myhostname=$(hostname -f)
-
-
-echo "Adding user ${mysqldbuser}@${mysqldbhost} and ${mysqldbuser}@localhost"
-expect <<EOF
-log_user 0
-# start mysql process using password prompt
-spawn mysql -u root -p
-expect "password:"
-send "${mysqlqdminpassword}\r"
-# echo all output until the end
-expect "mysql>"
-send "CREATE USER '${mysqldbuser}'@'${mysqldbhost}' IDENTIFIED BY '${mysqldbpasswd}';\r"
-expect "mysql>"
-send "CREATE USER '${mysqldbuser}'@'localhost' IDENTIFIED BY '${mysqldbpasswd}';\r"
-expect "mysql>"
-send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'%' IDENTIFIED BY '${mysqldbpasswd}';\r"
-log_user 1
-expect "mysql>"
-send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'${mysqldbhost}';\r"
-expect "mysql>"
-send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'localhost';\r"
-expect "mysql>"
-send "flush privileges;\r"
-send "\q"
-EOF
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/bro_index.template
deleted file mode 100644
index 951d967..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/bro_index.template
+++ /dev/null
@@ -1,217 +0,0 @@
-{
- "template": "bro_index*",
- "mappings": {
- "bro_doc": {
- "_timestamp": {
- "enabled": true
- },
- "dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
- }
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
- }
- }
- ],
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "source:type": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ip_dst_addr": {
- "type": "ip"
- },
- "ip_dst_port": {
- "type": "integer"
- },
- "ip_src_addr": {
- "type": "ip"
- },
- "ip_src_port": {
- "type": "integer"
- },
- "status_code": {
- "type": "integer"
- },
- "method": {
- "type": "string",
- "index": "not_analyzed"
- },
- "protocol": {
- "type": "string",
- "index": "not_analyzed"
- },
- "request_body_len": {
- "type": "integer"
- },
- "uri": {
- "type": "string",
- "index": "not_analyzed"
- },
- "uid": {
- "type": "string",
- "index": "not_analyzed"
- },
- "referrer": {
- "type": "string",
- "index": "not_analyzed"
- },
- "trans_depth": {
- "type": "integer"
- },
- "host": {
- "type": "string",
- "index": "not_analyzed"
- },
- "status_msg": {
- "type": "string",
- "index": "not_analyzed"
- },
- "response_body_len": {
- "type": "integer"
- },
- "user_agent": {
- "type": "string"
- },
- "query": {
- "type": "string",
- "index": "not_analyzed"
- },
- "answers": {
- "type": "ip"
- },
- "AA": {
- "type": "boolean"
- },
- "TC": {
- "type": "boolean"
- },
- "RA": {
- "type": "boolean"
- },
- "RD": {
- "type": "boolean"
- },
- "rejected": {
- "type": "boolean"
- },
- "qclass_name": {
- "type": "string",
- "index": "not_analyzed"
- },
- "proto": {
- "type": "string",
- "index": "not_analyzed"
- },
- "rcode": {
- "type": "integer"
- },
- "rcode_name": {
- "type": "string",
- "index": "not_analyzed"
- },
- "trans_id": {
- "type": "integer"
- },
- "Z": {
- "type": "integer"
- },
- "qclass": {
- "type": "integer"
- },
- "qtype": {
- "type": "integer"
- },
- "qtype_name": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/createMysqlGeoIp.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/createMysqlGeoIp.sh
deleted file mode 100755
index 40f11e1..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/createMysqlGeoIp.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-from ../scripts/params import params
-
-geoipscript=$1
-geoipurl=$2
-mysqlqdminpassword=$3
-
-# Download and extract the actual GeoIP files
-mkdir -p /tmp/geoip
-mkdir -p /var/lib/mysql-files/
-
-# Stage the GeoIP data
-pushd /tmp/geoip
-curl -O ${geoipurl}
-tar xf GeoLiteCity-latest.tar.xz
-cp /tmp/geoip/*/*.csv /var/lib/mysql-files/
-popd
-
-# Load MySQL with the GeoIP data
-expect <<EOF
-log_user 0
-# start mysql process using password prompt
-spawn mysql -u root -p
-expect "password:"
-send "${mysqlqdminpassword}\r"
-# echo all output until the end
-log_user 1
-expect "mysql>"
-send "source ${geoipscript}\r"
-expect "mysql>"
-send "show databases;\r"
-send "\q"
-EOF
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/removeMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/removeMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/removeMysqlUser.sh
deleted file mode 100755
index 3fc7afb..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/removeMysqlUser.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-
-mysqldbuser=$1
-userhost=$2
-mysqlqdminpassword=$3
-myhostname=$(hostname -f)
-sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E"
-
-echo "Removing user $mysqldbuser@$userhost"
-expect <<EOF
-log_user 0
-# start mysql process using password prompt
-spawn mysql -u root -p
-expect "password:"
-send "${mysqlqdminpassword}\r"
-# echo all output until the end
-log_user 1
-expect "mysql>"
-send "DROP USER '${mysqldbuser}'@'${userhost}';\r"
-expect "mysql>"
-send "flush privileges;;\r"
-send "\q"
-EOF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/snort_index.template
deleted file mode 100644
index bf943df..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/snort_index.template
+++ /dev/null
@@ -1,183 +0,0 @@
-{
- "template": "snort_index*",
- "mappings": {
- "snort_doc": {
- "_timestamp": {
- "enabled": true
- },
- "dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
- }
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
- }
- }
- ],
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "source:type": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ip_dst_addr": {
- "type": "ip"
- },
- "ip_dst_port": {
- "type": "integer"
- },
- "ip_src_addr": {
- "type": "ip"
- },
- "ip_src_port": {
- "type": "integer"
- },
- "dgmlen": {
- "type": "integer"
- },
- "ethdst": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ethlen": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ethsrc": {
- "type": "string",
- "index": "not_analyzed"
- },
- "id": {
- "type": "integer"
- },
- "iplen": {
- "type": "integer"
- },
- "is_alert": {
- "type": "boolean"
- },
- "msg": {
- "type": "string"
- },
- "protocol": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sig_generator": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sig_id": {
- "type": "integer"
- },
- "sig_rev": {
- "type": "string"
- },
- "tcpack": {
- "type": "string"
- },
- "tcpflags": {
- "type": "string"
- },
- "tcpseq": {
- "type": "string"
- },
- "tcpwindow": {
- "type": "string"
- },
- "threat:triage:level": {
- "type": "double"
- },
- "tos": {
- "type": "integer"
- },
- "ttl": {
- "type": "integer"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/yaf_index.template
deleted file mode 100644
index 7743afc..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/yaf_index.template
+++ /dev/null
@@ -1,205 +0,0 @@
-{
- "template": "yaf_index*",
- "mappings": {
- "yaf_doc": {
- "_timestamp": {
- "enabled": true
- },
- "dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
- }
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
- }
- }
- ],
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "source:type": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ip_dst_addr": {
- "type": "ip"
- },
- "ip_dst_port": {
- "type": "integer"
- },
- "ip_src_addr": {
- "type": "ip"
- },
- "ip_src_port": {
- "type": "integer"
- },
- "start_time": {
- "type": "date",
- "format": "epoch_millis"
- },
- "end_time": {
- "type": "date",
- "format": "epoch_millis"
- },
- "duration": {
- "type": "double"
- },
- "rtt": {
- "type": "double"
- },
- "proto": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sip": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sp": {
- "type": "string",
- "index": "not_analyzed"
- },
- "dip": {
- "type": "string",
- "index": "not_analyzed"
- },
- "dp": {
- "type": "string",
- "index": "not_analyzed"
- },
- "iflags": {
- "type": "string",
- "index": "not_analyzed"
- },
- "uflags": {
- "type": "string",
- "index": "not_analyzed"
- },
- "riflags": {
- "type": "string"
- },
- "ruflags": {
- "type": "string",
- "index": "not_analyzed"
- },
- "isn": {
- "type": "string",
- "index": "not_analyzed"
- },
- "risn": {
- "type": "string",
- "index": "not_analyzed"
- },
- "tag": {
- "type": "string",
- "index": "not_analyzed"
- },
- "rtag": {
- "type": "string",
- "index": "not_analyzed"
- },
- "pkt": {
- "type": "integer"
- },
- "oct": {
- "type": "integer"
- },
- "rpkt": {
- "type": "integer"
- },
- "roct": {
- "type": "integer"
- },
- "app": {
- "type": "string",
- "index": "not_analyzed"
- },
- "end-reason": {
- "type": "string"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_commands.py
deleted file mode 100755
index 026c316..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_commands.py
+++ /dev/null
@@ -1,174 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import os
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-
-import metron_service
-
-
-# Wrap major operations and functionality in this class
-class EnrichmentCommands:
- __params = None
- __enrichment_topology = None
- __enrichment_topic = None
- __enrichment_error_topic = None
- __threat_intel_error_topic = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__enrichment_topology = params.metron_enrichment_topology
- self.__enrichment_topic = params.metron_enrichment_topic
- self.__enrichment_error_topic = params.metron_enrichment_error_topic
- self.__threat_intel_error_topic = params.metron_threat_intel_error_topic
- self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file)
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.enrichment_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- def setup_repo(self):
-
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
-
- def remote_repo():
- Logger.info('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
-
- repo_type = self.__params.yum_repo_type
-
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- Logger.info("Writing out repo file")
- repo_template = ("echo \"[METRON-0.3.0]\n"
- "name=Metron 0.3.0 packages\n"
- "baseurl={0}\n"
- "gpgcheck=0\n"
- "enabled=1\n\""
- " > /etc/yum.repos.d/metron.repo")
- Execute(repo_template.format(self.__params.repo_url))
- else:
- raise ValueError("Unsupported repo type '{0}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{0}/kafka-topics.sh \
- --zookeeper {1} \
- --create \
- --topic {2} \
- --partitions {3} \
- --replication-factor {4} \
- --config retention.bytes={5}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = int(self.__params.metron_topic_retention)
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
-
- Logger.info("Creating topics for enrichment")
- topics = [self.__enrichment_topic, self.__enrichment_error_topic, self.__threat_intel_error_topic]
- for topic in topics:
- Logger.info("Creating topic'{0}'".format(topic))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- topic,
- num_partitions,
- replication_factor,
- retention_bytes))
-
- Logger.info("Done creating Kafka topics")
-
- def start_enrichment_topology(self):
- Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
- start_cmd_template = """{0}/bin/start_enrichment_topology.sh \
- -s {1} \
- -z {2}"""
- Logger.info('Starting ' + self.__enrichment_topology)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum))
-
- Logger.info('Finished starting enrichment topology')
-
- def stop_enrichment_topology(self):
- Logger.info('Stopping ' + self.__enrichment_topology)
- stop_cmd = 'storm kill ' + self.__enrichment_topology
- Execute(stop_cmd)
- Logger.info('Done stopping enrichment topologies')
-
- def restart_enrichment_topology(self, env):
- Logger.info('Restarting the enrichment topologies')
- self.stop_enrichment_topology()
-
- # Wait for old topology to be cleaned up by Storm, before starting again.
- retries = 0
- topology_active = self.is_topology_active(env)
- while topology_active and retries < 3:
- Logger.info('Existing topology still active. Will wait and retry')
- time.sleep(40)
- topology_active = self.is_topology_active(env)
- retries += 1
-
- if not topology_active:
- self.start_enrichment_topology()
- Logger.info('Done restarting the enrichment topology')
- else:
- Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
-
- def is_topology_active(self, env):
- env.set_params(self.__params)
-
- active = True
- topologies = metron_service.get_running_topologies()
- is_running = False
- if self.__enrichment_topology in topologies:
- is_running = topologies[self.__enrichment_topology] in ['ACTIVE', 'REBALANCING']
- active &= is_running
- return active
-
- def create_hbase_tables(self):
- add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf)
- Execute(add_enrichment_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
- )
-
- add_threatintel_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.threatintel_table, self.__params.threatintel_cf)
- Execute(add_threatintel_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_master.py
deleted file mode 100755
index d9da4f6..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_master.py
+++ /dev/null
@@ -1,86 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import File
-from resource_management.core.source import Template
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.script import Script
-
-from enrichment_commands import EnrichmentCommands
-
-
-class Enrichment(Script):
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.setup_repo()
- self.install_packages(env)
- self.configure(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
-
- File(format("{metron_config_path}/enrichment.properties"),
- content=Template("enrichment.properties.j2"),
- owner=params.metron_user,
- group=params.metron_group
- )
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
-
- if not commands.is_configured():
- commands.init_kafka_topics()
- commands.create_hbase_tables()
- commands.set_configured()
-
- commands.start_enrichment_topology()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.stop_enrichment_topology()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = EnrichmentCommands(status_params)
-
- if not commands.is_topology_active(env):
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.restart_enrichment_topology(env)
-
- def kafkabuild(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.init_kafka_topics()
-
-
-if __name__ == "__main__":
- Enrichment().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_commands.py
deleted file mode 100755
index 297ede6..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_commands.py
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import os
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-
-import metron_service
-
-
-# Wrap major operations and functionality in this class
-class IndexingCommands:
- __params = None
- __indexing = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__indexing = params.metron_indexing_topology
- self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file)
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.indexing_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- def setup_repo(self):
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
- Execute("echo \"[METRON-0.3.0]\n"
- "name=Metron 0.3.0 packages\n"
- "baseurl=file:///localrepo\n"
- "gpgcheck=0\n"
- "enabled=1\" > /etc/yum.repos.d/local.repo")
-
- def remote_repo():
- print('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
- repo_type = self.__params.yum_repo_type
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- else:
- raise ValueError("Unsupported repo type '{0}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{0}/kafka-topics.sh \
- --zookeeper {1} \
- --create \
- --topic {2} \
- --partitions {3} \
- --replication-factor {4} \
- --config retention.bytes={5}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = int(self.__params.metron_topic_retention)
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating topics for indexing")
-
- Logger.info("Creating topic'{0}'".format(self.__indexing))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- self.__indexing,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def init_hdfs_dir(self):
- Logger.info('Creating HDFS indexing directory')
- self.__params.HdfsResource(self.__params.metron_apps_indexed_hdfs_dir,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- group=self.__params.user_group,
- mode=0775,
- )
- Logger.info('Done creating HDFS indexing directory')
-
-
- def start_indexing_topology(self):
- Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing))
- start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \
- -s {1} \
- -z {2}"""
- Logger.info('Starting ' + self.__indexing)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum))
-
- Logger.info('Finished starting indexing topology')
-
- def stop_indexing_topology(self):
- Logger.info('Stopping ' + self.__indexing)
- stop_cmd = 'storm kill ' + self.__indexing
- Execute(stop_cmd)
- Logger.info('Done stopping indexing topologies')
-
- def restart_indexing_topology(self, env):
- Logger.info('Restarting the indexing topologies')
- self.stop_indexing_topology()
-
- # Wait for old topology to be cleaned up by Storm, before starting again.
- retries = 0
- topology_active = self.is_topology_active(env)
- while self.is_topology_active(env) and retries < 3:
- Logger.info('Existing topology still active. Will wait and retry')
- time.sleep(10)
- retries += 1
-
- if not topology_active:
- Logger.info('Waiting for storm kill to complete')
- time.sleep(30)
- self.start_indexing_topology()
- Logger.info('Done restarting the indexing topologies')
- else:
- Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
-
- def is_topology_active(self, env):
- env.set_params(self.__params)
- active = True
- topologies = metron_service.get_running_topologies()
- is_running = False
- if self.__indexing in topologies:
- is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING']
- active &= is_running
- return active
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_master.py
deleted file mode 100755
index 43a62f9..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_master.py
+++ /dev/null
@@ -1,120 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute
-from resource_management.core.resources.system import File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions import format as ambari_format
-from resource_management.libraries.script import Script
-
-import metron_service
-from indexing_commands import IndexingCommands
-
-
-class Indexing(Script):
- __configured = False
-
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = IndexingCommands(params)
- commands.setup_repo()
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
-
- commands = IndexingCommands(params)
- metron_service.load_global_config(params)
-
- if not commands.is_configured():
- commands.init_kafka_topics()
- commands.init_hdfs_dir()
- commands.set_configured()
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = IndexingCommands(params)
- commands.start_indexing_topology()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = IndexingCommands(params)
- commands.stop_indexing_topology()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = IndexingCommands(status_params)
- if not commands.is_topology_active(env):
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = IndexingCommands(params)
- commands.restart_indexing_topology(env)
-
- def elasticsearch_template_install(self, env):
- from params import params
- env.set_params(params)
-
- File(params.bro_index_path,
- mode=0755,
- content=StaticFile('bro_index.template')
- )
-
- File(params.snort_index_path,
- mode=0755,
- content=StaticFile('snort_index.template')
- )
-
- File(params.yaf_index_path,
- mode=0755,
- content=StaticFile('yaf_index.template')
- )
-
- bro_cmd = ambari_format(
- 'curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}')
- Execute(bro_cmd, logoutput=True)
- snort_cmd = ambari_format(
- 'curl -s -XPOST http://{es_http_url}/_template/snort_index -d @{snort_index_path}')
- Execute(snort_cmd, logoutput=True)
- yaf_cmd = ambari_format(
- 'curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}')
- Execute(yaf_cmd, logoutput=True)
-
- def elasticsearch_template_delete(self, env):
- from params import params
- env.set_params(params)
-
- bro_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/bro_index*"')
- Execute(bro_cmd, logoutput=True)
- snort_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/snort_index*"')
- Execute(snort_cmd, logoutput=True)
- yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/yaf_index*"')
- Execute(yaf_cmd, logoutput=True)
-
-
-if __name__ == "__main__":
- Indexing().execute()