You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2016/12/13 18:25:44 UTC
[2/4] incubator-metron git commit: METRON-580: Remove hard-coded
Metron version from Ambari MPack code (mmiklavc) closes
apache/incubator-metron#364
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
new file mode 100644
index 0000000..2677f60
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -0,0 +1,295 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>METRON</name>
+ <displayName>Metron</displayName>
+ <comment>A scalable advanced security analytics framework built on Hadoop</comment>
+ <version>${metron.version}</version>
+ <components>
+
+ <component>
+ <name>METRON_PARSERS</name>
+ <displayName>Metron Parsers</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <reassignAllowed>false</reassignAllowed>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>STORM/SUPERVISOR</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/parser_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>METRON_ENRICHMENT_MASTER</name>
+ <displayName>Metron Enrichment</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>HBASE/HBASE_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/enrichment_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>METRON_ENRICHMENT_MYSQL_SERVER</name>
+ <displayName>MySQL Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <commandScript>
+ <script>scripts/mysql_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>CLEAN</name>
+ <commandScript>
+ <script>scripts/mysql_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ </component>
+
+ <component>
+ <name>METRON_INDEXING</name>
+ <displayName>Metron Indexing</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <reassignAllowed>false</reassignAllowed>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <dependencies>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>STORM/SUPERVISOR</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>ELASTICSEARCH_TEMPLATE_INSTALL</name>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ <customCommand>
+ <name>ELASTICSEARCH_TEMPLATE_DELETE</name>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ </component>
+ </components>
+
+ <themes>
+ <theme>
+ <fileName>metron_theme.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>metron-common</name>
+ </package>
+ <package>
+ <name>metron-data-management</name>
+ </package>
+ <package>
+ <name>metron-parsers</name>
+ </package>
+ <package>
+ <name>metron-enrichment</name>
+ </package>
+ <package>
+ <name>metron-indexing</name>
+ </package>
+ <package>
+ <name>metron-elasticsearch</name>
+ </package>
+ <package>
+ <name>metron-pcap</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat6,redhat7</osFamily>
+ <packages>
+ <package>
+ <name>expect</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat7</osFamily>
+ <packages>
+ <package>
+ <name>mysql-community-release</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>mysql-community-server</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>MySQL-python</package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat6</osFamily>
+ <packages>
+ <package>
+ <name>mysql-server</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>mysql</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ <service>KAFKA</service>
+ <service>STORM</service>
+ <service>ZOOKEEPER</service>
+ <service>HBASE</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>metron-env</config-type>
+ <config-type>storm-env</config-type>
+ <config-type>storm-site</config-type>
+ <config-type>kafka-broker</config-type>
+ <config-type>kafka-env</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/addMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/addMysqlUser.sh
new file mode 100755
index 0000000..8a6a788
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/addMysqlUser.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+mysqldbuser=$1
+mysqldbpasswd=$2
+mysqldbhost=$3
+mysqlqdminpassword=$4
+myhostname=$(hostname -f)
+
+
+echo "Adding user ${mysqldbuser}@${mysqldbhost} and ${mysqldbuser}@localhost"
+expect <<EOF
+log_user 0
+# start mysql process using password prompt
+spawn mysql -u root -p
+expect "password:"
+send "${mysqlqdminpassword}\r"
+# echo all output until the end
+expect "mysql>"
+send "CREATE USER '${mysqldbuser}'@'${mysqldbhost}' IDENTIFIED BY '${mysqldbpasswd}';\r"
+expect "mysql>"
+send "CREATE USER '${mysqldbuser}'@'localhost' IDENTIFIED BY '${mysqldbpasswd}';\r"
+expect "mysql>"
+send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'%' IDENTIFIED BY '${mysqldbpasswd}';\r"
+log_user 1
+expect "mysql>"
+send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'${mysqldbhost}';\r"
+expect "mysql>"
+send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'localhost';\r"
+expect "mysql>"
+send "flush privileges;\r"
+send "\q"
+EOF
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
new file mode 100644
index 0000000..951d967
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -0,0 +1,217 @@
+{
+ "template": "bro_index*",
+ "mappings": {
+ "bro_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "status_code": {
+ "type": "integer"
+ },
+ "method": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "protocol": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "request_body_len": {
+ "type": "integer"
+ },
+ "uri": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "uid": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "referrer": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "trans_depth": {
+ "type": "integer"
+ },
+ "host": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "status_msg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "response_body_len": {
+ "type": "integer"
+ },
+ "user_agent": {
+ "type": "string"
+ },
+ "query": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "answers": {
+ "type": "ip"
+ },
+ "AA": {
+ "type": "boolean"
+ },
+ "TC": {
+ "type": "boolean"
+ },
+ "RA": {
+ "type": "boolean"
+ },
+ "RD": {
+ "type": "boolean"
+ },
+ "rejected": {
+ "type": "boolean"
+ },
+ "qclass_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "proto": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "rcode": {
+ "type": "integer"
+ },
+ "rcode_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "trans_id": {
+ "type": "integer"
+ },
+ "Z": {
+ "type": "integer"
+ },
+ "qclass": {
+ "type": "integer"
+ },
+ "qtype": {
+ "type": "integer"
+ },
+ "qtype_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/createMysqlGeoIp.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/createMysqlGeoIp.sh
new file mode 100755
index 0000000..40f11e1
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/createMysqlGeoIp.sh
@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+from ../scripts/params import params
+
+geoipscript=$1
+geoipurl=$2
+mysqlqdminpassword=$3
+
+# Download and extract the actual GeoIP files
+mkdir -p /tmp/geoip
+mkdir -p /var/lib/mysql-files/
+
+# Stage the GeoIP data
+pushd /tmp/geoip
+curl -O ${geoipurl}
+tar xf GeoLiteCity-latest.tar.xz
+cp /tmp/geoip/*/*.csv /var/lib/mysql-files/
+popd
+
+# Load MySQL with the GeoIP data
+expect <<EOF
+log_user 0
+# start mysql process using password prompt
+spawn mysql -u root -p
+expect "password:"
+send "${mysqlqdminpassword}\r"
+# echo all output until the end
+log_user 1
+expect "mysql>"
+send "source ${geoipscript}\r"
+expect "mysql>"
+send "show databases;\r"
+send "\q"
+EOF
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/removeMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/removeMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/removeMysqlUser.sh
new file mode 100755
index 0000000..3fc7afb
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/removeMysqlUser.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+mysqldbuser=$1
+userhost=$2
+mysqlqdminpassword=$3
+myhostname=$(hostname -f)
+sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E"
+
+echo "Removing user $mysqldbuser@$userhost"
+expect <<EOF
+log_user 0
+# start mysql process using password prompt
+spawn mysql -u root -p
+expect "password:"
+send "${mysqlqdminpassword}\r"
+# echo all output until the end
+log_user 1
+expect "mysql>"
+send "DROP USER '${mysqldbuser}'@'${userhost}';\r"
+expect "mysql>"
+send "flush privileges;;\r"
+send "\q"
+EOF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
new file mode 100644
index 0000000..bf943df
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -0,0 +1,183 @@
+{
+ "template": "snort_index*",
+ "mappings": {
+ "snort_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "dgmlen": {
+ "type": "integer"
+ },
+ "ethdst": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ethlen": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ethsrc": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "iplen": {
+ "type": "integer"
+ },
+ "is_alert": {
+ "type": "boolean"
+ },
+ "msg": {
+ "type": "string"
+ },
+ "protocol": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sig_generator": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sig_id": {
+ "type": "integer"
+ },
+ "sig_rev": {
+ "type": "string"
+ },
+ "tcpack": {
+ "type": "string"
+ },
+ "tcpflags": {
+ "type": "string"
+ },
+ "tcpseq": {
+ "type": "string"
+ },
+ "tcpwindow": {
+ "type": "string"
+ },
+ "threat:triage:level": {
+ "type": "double"
+ },
+ "tos": {
+ "type": "integer"
+ },
+ "ttl": {
+ "type": "integer"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
new file mode 100644
index 0000000..7743afc
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
@@ -0,0 +1,205 @@
+{
+ "template": "yaf_index*",
+ "mappings": {
+ "yaf_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "start_time": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "end_time": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "duration": {
+ "type": "double"
+ },
+ "rtt": {
+ "type": "double"
+ },
+ "proto": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "dip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "dp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "iflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "uflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "riflags": {
+ "type": "string"
+ },
+ "ruflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "isn": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "risn": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "tag": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "rtag": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "pkt": {
+ "type": "integer"
+ },
+ "oct": {
+ "type": "integer"
+ },
+ "rpkt": {
+ "type": "integer"
+ },
+ "roct": {
+ "type": "integer"
+ },
+ "app": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "end-reason": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
new file mode 100755
index 0000000..cb9cf22
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class EnrichmentCommands:
+ __params = None
+ __enrichment_topology = None
+ __enrichment_topic = None
+ __enrichment_error_topic = None
+ __threat_intel_error_topic = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__enrichment_topology = params.metron_enrichment_topology
+ self.__enrichment_topic = params.metron_enrichment_topic
+ self.__enrichment_error_topic = params.metron_enrichment_error_topic
+ self.__threat_intel_error_topic = params.metron_threat_intel_error_topic
+ self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.enrichment_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+
+ def remote_repo():
+ Logger.info('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+
+ repo_type = self.__params.yum_repo_type
+
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ Logger.info("Writing out repo file")
+ repo_template = ("echo \"[METRON-${metron.version}]\n"
+ "name=Metron ${metron.version} packages\n"
+ "baseurl={0}\n"
+ "gpgcheck=0\n"
+ "enabled=1\n\""
+ " > /etc/yum.repos.d/metron.repo")
+ Execute(repo_template.format(self.__params.repo_url))
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+
+ Logger.info("Creating topics for enrichment")
+ topics = [self.__enrichment_topic, self.__enrichment_error_topic, self.__threat_intel_error_topic]
+ for topic in topics:
+ Logger.info("Creating topic'{0}'".format(topic))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ topic,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+
+ Logger.info("Done creating Kafka topics")
+
+ def start_enrichment_topology(self):
+ Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
+ start_cmd_template = """{0}/bin/start_enrichment_topology.sh \
+ -s {1} \
+ -z {2}"""
+ Logger.info('Starting ' + self.__enrichment_topology)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting enrichment topology')
+
+ def stop_enrichment_topology(self):
+ Logger.info('Stopping ' + self.__enrichment_topology)
+ stop_cmd = 'storm kill ' + self.__enrichment_topology
+ Execute(stop_cmd)
+ Logger.info('Done stopping enrichment topologies')
+
+ def restart_enrichment_topology(self, env):
+ Logger.info('Restarting the enrichment topologies')
+ self.stop_enrichment_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while topology_active and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(40)
+ topology_active = self.is_topology_active(env)
+ retries += 1
+
+ if not topology_active:
+ self.start_enrichment_topology()
+ Logger.info('Done restarting the enrichment topology')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+
+ active = True
+ topologies = metron_service.get_running_topologies()
+ is_running = False
+ if self.__enrichment_topology in topologies:
+ is_running = topologies[self.__enrichment_topology] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
+
+ def create_hbase_tables(self):
+ add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf)
+ Execute(add_enrichment_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
+
+ add_threatintel_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.threatintel_table, self.__params.threatintel_cf)
+ Execute(add_threatintel_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
new file mode 100755
index 0000000..d9da4f6
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
@@ -0,0 +1,86 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import File
+from resource_management.core.source import Template
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script import Script
+
+from enrichment_commands import EnrichmentCommands
+
+
+class Enrichment(Script):
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.setup_repo()
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ File(format("{metron_config_path}/enrichment.properties"),
+ content=Template("enrichment.properties.j2"),
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.create_hbase_tables()
+ commands.set_configured()
+
+ commands.start_enrichment_topology()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.stop_enrichment_topology()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = EnrichmentCommands(status_params)
+
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.restart_enrichment_topology(env)
+
+ def kafkabuild(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.init_kafka_topics()
+
+
+if __name__ == "__main__":
+ Enrichment().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
new file mode 100755
index 0000000..8ed3f9a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class IndexingCommands:
+ __params = None
+ __indexing = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__indexing = params.metron_indexing_topology
+ self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.indexing_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-${metron.version}]\n"
+ "name=Metron ${metron.version} packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating topics for indexing")
+
+ Logger.info("Creating topic'{0}'".format(self.__indexing))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ self.__indexing,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def init_hdfs_dir(self):
+ Logger.info('Creating HDFS indexing directory')
+ self.__params.HdfsResource(self.__params.metron_apps_indexed_hdfs_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.user_group,
+ mode=0775,
+ )
+ Logger.info('Done creating HDFS indexing directory')
+
+
+ def start_indexing_topology(self):
+ Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing))
+ start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \
+ -s {1} \
+ -z {2}"""
+ Logger.info('Starting ' + self.__indexing)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting indexing topology')
+
+ def stop_indexing_topology(self):
+ Logger.info('Stopping ' + self.__indexing)
+ stop_cmd = 'storm kill ' + self.__indexing
+ Execute(stop_cmd)
+ Logger.info('Done stopping indexing topologies')
+
+ def restart_indexing_topology(self, env):
+ Logger.info('Restarting the indexing topologies')
+ self.stop_indexing_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while self.is_topology_active(env) and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(10)
+ retries += 1
+
+ if not topology_active:
+ Logger.info('Waiting for storm kill to complete')
+ time.sleep(30)
+ self.start_indexing_topology()
+ Logger.info('Done restarting the indexing topologies')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+ active = True
+ topologies = metron_service.get_running_topologies()
+ is_running = False
+ if self.__indexing in topologies:
+ is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
new file mode 100755
index 0000000..43a62f9
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -0,0 +1,120 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.script import Script
+
+import metron_service
+from indexing_commands import IndexingCommands
+
+
+class Indexing(Script):
+ __configured = False
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = IndexingCommands(params)
+ commands.setup_repo()
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ commands = IndexingCommands(params)
+ metron_service.load_global_config(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.init_hdfs_dir()
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = IndexingCommands(params)
+ commands.start_indexing_topology()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = IndexingCommands(params)
+ commands.stop_indexing_topology()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = IndexingCommands(status_params)
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = IndexingCommands(params)
+ commands.restart_indexing_topology(env)
+
+ def elasticsearch_template_install(self, env):
+ from params import params
+ env.set_params(params)
+
+ File(params.bro_index_path,
+ mode=0755,
+ content=StaticFile('bro_index.template')
+ )
+
+ File(params.snort_index_path,
+ mode=0755,
+ content=StaticFile('snort_index.template')
+ )
+
+ File(params.yaf_index_path,
+ mode=0755,
+ content=StaticFile('yaf_index.template')
+ )
+
+ bro_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}')
+ Execute(bro_cmd, logoutput=True)
+ snort_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_http_url}/_template/snort_index -d @{snort_index_path}')
+ Execute(snort_cmd, logoutput=True)
+ yaf_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}')
+ Execute(yaf_cmd, logoutput=True)
+
+ def elasticsearch_template_delete(self, env):
+ from params import params
+ env.set_params(params)
+
+ bro_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/bro_index*"')
+ Execute(bro_cmd, logoutput=True)
+ snort_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/snort_index*"')
+ Execute(snort_cmd, logoutput=True)
+ yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/yaf_index*"')
+ Execute(yaf_cmd, logoutput=True)
+
+
+if __name__ == "__main__":
+ Indexing().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
new file mode 100644
index 0000000..57da2c7
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -0,0 +1,76 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import json
+import subprocess
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Directory, File
+from resource_management.core.resources.system import Execute
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions import format as ambari_format
+
+
+def init_config():
+ Logger.info('Loading config into ZooKeeper')
+ Execute(ambari_format(
+ "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
+ path=ambari_format("{java_home}/bin")
+ )
+
+
+def get_running_topologies():
+ Logger.info('Getting Running Storm Topologies from Storm REST Server')
+
+ cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary')
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+
+ try:
+ stormjson = json.loads(stdout)
+ except ValueError:
+ return {}
+
+ topologiesDict = {}
+
+ for topology in stormjson['topologies']:
+ topologiesDict[topology['name']] = topology['status']
+
+ Logger.info("Topologies: " + str(topologiesDict))
+ return topologiesDict
+
+
+def load_global_config(params):
+ Logger.info('Create Metron Local Config Directory')
+ Logger.info("Configure Metron global.json")
+
+ directories = [params.metron_zookeeper_config_path]
+ Directory(directories,
+ mode=0755,
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
+ File("{0}/global.json".format(params.metron_zookeeper_config_path),
+ owner=params.metron_user,
+ content=InlineTemplate(params.global_json_template)
+ )
+
+ File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'),
+ owner=params.metron_user,
+ content=InlineTemplate(params.global_properties_template))
+
+ init_config()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_server.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_server.py
new file mode 100755
index 0000000..fcc83af
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_server.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.script import Script
+
+import mysql_users
+from mysql_service import mysql_service
+from mysql_utils import mysql_configure
+from resource_management.core.resources.packaging import Package
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+class MysqlServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def clean(self, env):
+ from params import params
+ env.set_params(params)
+ if params.install_mysql == 'Yes':
+ mysql_users.mysql_deluser()
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+ mysql_configure()
+
+ def start(self, env, rolling_restart=False):
+ from params import params
+ env.set_params(params)
+ mysql_service(daemon_name=params.daemon_name, action='start')
+
+ def stop(self, env, rolling_restart=False):
+ from params import params
+ env.set_params(params)
+ mysql_service(daemon_name=params.daemon_name, action='stop')
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+
+ mysql_service(daemon_name=status_params.daemon_name, action='status')
+
+
+if __name__ == "__main__":
+ MysqlServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_service.py
new file mode 100755
index 0000000..2e0ce8b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_service.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning, Fail
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions.format import format
+
+
+def mysql_service(daemon_name=None, action='start'):
+ status_cmd = format("pgrep -l '^{mysql_process_name}$'")
+ cmd = ('service', daemon_name, action)
+
+ if action == 'status':
+ try:
+ Execute(status_cmd)
+ except Fail:
+ raise ComponentIsNotRunning()
+ elif action == 'stop':
+ Execute(cmd,
+ logoutput=True,
+ only_if=status_cmd,
+ sudo=True,
+ )
+ elif action == 'start':
+ Execute(cmd,
+ logoutput=True,
+ not_if=status_cmd,
+ sudo=True,
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_users.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_users.py
new file mode 100755
index 0000000..b778e85
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_users.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+
+
+# Used to add metron access to the needed components
+def mysql_adduser():
+ from params import params
+
+ File(params.mysql_adduser_path,
+ mode=0755,
+ content=StaticFile('addMysqlUser.sh')
+ )
+
+ add_user_cmd = format("bash -x {mysql_adduser_path} {metron_user} {enrichment_metron_user_passwd!p} {mysql_host} {mysql_admin_password!p}")
+ Execute(add_user_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
+
+
+# Removes hive metron from components
+def mysql_deluser():
+ from params import params
+
+ File(params.mysql_deluser_path,
+ mode=0755,
+ content=StaticFile('removeMysqlUser.sh')
+ )
+
+ del_user_cmd = format("bash -x {mysql_deluser_path} {metron_user} {mysql_host} {mysql_admin_password!p}")
+ Execute(del_user_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_utils.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_utils.py
new file mode 100755
index 0000000..63cdb38
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_utils.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+from mysql_service import mysql_service
+
+import mysql_users
+
+
+def mysql_configure():
+ from params import params
+
+ if params.install_mysql == 'Yes':
+ mysql_service(daemon_name=params.daemon_name, action='start')
+
+ replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname)
+ Execute(replace_bind_address,
+ sudo=True,
+ )
+
+ mysql_users.mysql_adduser()
+
+ File(params.mysql_create_geoip_path,
+ mode=0755,
+ content=StaticFile('createMysqlGeoIp.sh')
+ )
+
+ geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {geoip_ddl} {geoip_url} {mysql_admin_password!p}")
+
+ Execute(geoip_setup_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ )
+
+ if params.install_mysql == 'Yes':
+ mysql_service(daemon_name=params.daemon_name, action='stop')
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/__init__.py
new file mode 100755
index 0000000..242460e
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/__init__.py
@@ -0,0 +1,18 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params.py
new file mode 100755
index 0000000..953435d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
new file mode 100755
index 0000000..6b3052d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import functools
+import os
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.script import Script
+
+import status_params
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+hostname = config['hostname']
+user_group = config['configurations']['cluster-env']['user_group']
+metron_home = status_params.metron_home
+parsers = status_params.parsers
+metron_ddl_dir = metron_home + '/ddl'
+geoip_ddl = metron_ddl_dir + '/geoip_ddl.sql'
+geoip_url = config['configurations']['metron-env']['geoip_url']
+metron_indexing_topology = status_params.metron_indexing_topology
+metron_user = config['configurations']['metron-env']['metron_user']
+metron_group = config['configurations']['metron-env']['metron_group']
+metron_config_path = metron_home + '/config'
+metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
+metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
+parsers_configured_flag_file = status_params.parsers_configured_flag_file
+enrichment_configured_flag_file = status_params.enrichment_configured_flag_file
+indexing_configured_flag_file = status_params.indexing_configured_flag_file
+global_json_template = config['configurations']['metron-env']['global-json']
+global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
+
+# Elasticsearch hosts and port management
+es_cluster_name = config['configurations']['metron-env']['es_cluster_name']
+es_hosts = config['configurations']['metron-env']['es_hosts']
+es_host_list = es_hosts.split(",")
+es_binary_port = config['configurations']['metron-env']['es_binary_port']
+es_url = ",".join([host + ":" + es_binary_port for host in es_host_list])
+es_http_port = config['configurations']['metron-env']['es_http_port']
+es_http_url = es_host_list[0] + ":" + es_http_port
+
+# install repo
+yum_repo_type = config['configurations']['metron-env']['repo_type']
+if yum_repo_type == 'local':
+ repo_url = 'file:///localrepo'
+else:
+ repo_url = config['configurations']['metron-env']['repo_url']
+
+# hadoop params
+stack_root = Script.get_stack_root()
+hadoop_home_dir = stack_select.get_hadoop_dir("home")
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+kafka_bin_dir = os.path.join(kafka_home, "bin")
+
+# zookeeper
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+has_zk_host = not len(zk_hosts) == 0
+zookeeper_quorum = None
+if has_zk_host:
+ if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+ else:
+ zookeeper_clientPort = '2181'
+ zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
+ # last port config
+ zookeeper_quorum += ':' + zookeeper_clientPort
+
+# Storm
+storm_rest_addr = status_params.storm_rest_addr
+
+# Kafka
+kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
+has_kafka_host = not len(kafka_hosts) == 0
+kafka_brokers = None
+if has_kafka_host:
+ if 'port' in config['configurations']['kafka-broker']:
+ kafka_broker_port = config['configurations']['kafka-broker']['port']
+ else:
+ kafka_broker_port = '6667'
+ kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
+ kafka_brokers += ':' + kafka_broker_port
+
+metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
+# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works
+# well enough until we find a better way via Ambari
+metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-env']['metron_apps_indexed_hdfs_dir']))
+metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
+
+local_grok_patterns_dir = format("{metron_home}/patterns")
+hdfs_grok_patterns_dir = format("{metron_apps_hdfs_dir}/patterns")
+
+# for create_hdfs_directory
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+dfs_type = default("/commandParams/dfs_type", "")
+
+# MYSQL
+if OSCheck.is_ubuntu_family():
+ mysql_configname = '/etc/mysql/my.cnf'
+else:
+ mysql_configname = '/etc/my.cnf'
+
+daemon_name = status_params.daemon_name
+
+install_mysql = config['configurations']['metron-env']['install_mysql']
+mysql_admin_password = config['configurations']['metron-env']['mysql_admin_password']
+
+# There will always be exactly one mysql_host
+mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][0]
+
+mysql_port = config['configurations']['metron-env']['metron_enrichment_db_port']
+
+mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
+mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
+mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
+
+enrichment_metron_user = config['configurations']['metron-env']['metron_enrichment_db_user']
+enrichment_metron_user_passwd = config['configurations']['metron-env']['metron_enrichment_db_password']
+enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(
+ enrichment_metron_user_passwd) else enrichment_metron_user_passwd
+mysql_process_name = status_params.mysql_process_name
+
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled=security_enabled,
+ keytab=hdfs_user_keytab,
+ kinit_path_local=kinit_path_local,
+ hadoop_bin_dir=hadoop_bin_dir,
+ hadoop_conf_dir=hadoop_conf_dir,
+ principal_name=hdfs_principal_name,
+ hdfs_site=hdfs_site,
+ default_fs=default_fs,
+ immutable_paths=get_not_managed_resources(),
+ dfs_type=dfs_type
+)
+
+# HBase
+enrichment_table = status_params.enrichment_table
+enrichment_cf = status_params.enrichment_cf
+threatintel_table = status_params.threatintel_table
+threatintel_cf = status_params.threatintel_cf
+
+metron_enrichment_topology = status_params.metron_enrichment_topology
+metron_enrichment_topic = status_params.metron_enrichment_topic
+metron_enrichment_error_topic = status_params.metron_enrichment_error_topic
+metron_threat_intel_error_topic = status_params.metron_threat_intel_error_topic
+
+# ES Templates
+bro_index_path = tmp_dir + "/bro_index.template"
+snort_index_path = tmp_dir + "/snort_index.template"
+yaf_index_path = tmp_dir + "/yaf_index.template"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_windows.py
new file mode 100755
index 0000000..4d11b35
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_windows.py
@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+raise NotImplementedError