You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/09/06 19:17:22 UTC
incubator-metron git commit: METRON-383: Create Ambari Service
Definition for Metron Parsers (mmiklavc via cestella) closes
apache/incubator-metron#218
Repository: incubator-metron
Updated Branches:
refs/heads/master fc2dc3d6b -> 47bc9b6de
METRON-383: Create Ambari Service Definition for Metron Parsers (mmiklavc via cestella) closes apache/incubator-metron#218
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/47bc9b6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/47bc9b6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/47bc9b6d
Branch: refs/heads/master
Commit: 47bc9b6defa3707b07ec1fa0123bbb72843a4b33
Parents: fc2dc3d
Author: mmiklavc <mi...@gmail.com>
Authored: Tue Sep 6 12:17:11 2016 -0700
Committer: cstella <ce...@gmail.com>
Committed: Tue Sep 6 12:17:11 2016 -0700
----------------------------------------------------------------------
metron-deployment/packaging/ambari/.gitignore | 3 +
.../0.2.0BETA/configuration/metron-parsers.xml | 57 +++++
.../PARSERS/0.2.0BETA/metainfo.xml | 108 +++++++++
.../0.2.0BETA/package/scripts/commands.py | 235 +++++++++++++++++++
.../package/scripts/params/__init__.py | 18 ++
.../0.2.0BETA/package/scripts/params/params.py | 32 +++
.../package/scripts/params/params_linux.py | 114 +++++++++
.../package/scripts/params/params_windows.py | 20 ++
.../package/scripts/params/status_params.py | 22 ++
.../0.2.0BETA/package/scripts/parser_master.py | 77 ++++++
.../0.2.0BETA/package/scripts/service_check.py | 40 ++++
.../package/templates/metron-global.json | 1 +
.../0.2.0BETA/quicklinks/quicklinks.json | 28 +++
.../PARSERS/0.2.0BETA/service_advisor.py | 68 ++++++
.../HDP/2.3/services/PARSERS/metainfo.xml | 27 +++
.../services/PARSERS/role_command_order.json | 17 ++
16 files changed, 867 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore
new file mode 100644
index 0000000..db8ded0
--- /dev/null
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -0,0 +1,3 @@
+archive.zip
+*.hash
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
new file mode 100755
index 0000000..71966ea
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>metron_home</name>
+ <value>/usr/metron/0.2.0BETA</value>
+ <description>Metron home directory</description>
+ <display-name>Metron home</display-name>
+ </property>
+ <property>
+ <name>parsers</name>
+ <value>bro,snort,yaf</value>
+ <description>Metron parsers to deploy</description>
+ <display-name>Metron parsers</display-name>
+ </property>
+ <property>
+ <name>metron_apps_hdfs_dir</name>
+ <value>/apps/metron</value>
+ <description>Metron apps HDFS dir</description>
+ <display-name>Metron apps HDFS dir</display-name>
+ </property>
+ <property>
+ <name>metron_zookeeper_config_dir</name>
+ <value>config/zookeeper</value>
+ <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
+ <display-name>Metron Zookeeper config dir</display-name>
+ </property>
+ <property>
+ <name>metron_user</name>
+ <value>metron</value>
+ <property-type>USER</property-type>
+ <description>The user for Metron</description>
+ <display-name>Metron User</display-name>
+ </property>
+ <property>
+ <name>metron_group</name>
+ <value>metron</value>
+ <description>The group for Metron</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
new file mode 100755
index 0000000..f1e5ab7
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
@@ -0,0 +1,108 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>PARSERS</name>
+ <displayName>Metron Parsers</displayName>
+ <comment>Parsers</comment>
+ <version>0.2.0BETA</version>
+
+ <components>
+ <component>
+ <name>PARSER_MASTER</name>
+ <displayName>Metron Parser Topologies</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/parser_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <configuration-dependencies>
+ <config-type>metron-parsers</config-type>
+ </configuration-dependencies>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>metron-common</name>
+ </package>
+ <package>
+ <name>metron-parsers</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ <service>KAFKA</service>
+ <service>STORM</service>
+ <service>ZOOKEEPER</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>metron-parsers</config-type>
+ </configuration-dependencies>
+
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
new file mode 100755
index 0000000..6b33e55
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
@@ -0,0 +1,235 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import subprocess
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import Template
+from resource_management.libraries.functions import format
+
+
+# Wrap major operations and functionality in this class
+class Commands:
+ __params = None
+ __parser_list = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__parser_list = self.__get_parsers(params)
+ self.__configured = os.path.isfile(self.__params.configured_flag_file)
+
+ # get list of parsers
+ def __get_parsers(self, params):
+ return params.parsers.replace(' ', '').split(',')
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ # Possible storm topology status states
+ # http://storm.apache.org/releases/0.10.0/javadocs/backtype/storm/generated/TopologyStatus.html
+ class StormStatus:
+ ACTIVE = "ACTIVE"
+ INACTIVE = "INACTIVE"
+ KILLED = "KILLED"
+ REBALANCING = "REBALANCING"
+
+ def init_parsers(self):
+ Logger.info(
+ "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir,
+ self.__params.metron_apps_dir))
+ self.__params.HdfsResource(self.__params.metron_apps_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ mode=0775,
+ source=self.__params.local_grok_patterns_dir)
+
+ Logger.info("Creating global.json file")
+ File(self.__params.metron_zookeeper_config_path + '/global.json',
+ content=Template("metron-global.json"),
+ owner=self.__params.metron_user,
+ mode=0775)
+ Logger.info("Done initializing parser configuration")
+
+ def get_parser_list(self):
+ return self.__parser_list
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.0BETA]\n"
+ "name=Metron 0.2.0BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{}/kafka-topics.sh \
+ --zookeeper {} \
+ --create \
+ --topic {} \
+ --partitions {} \
+ --replication-factor {} \
+ --config retention.bytes={}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = 10
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating main topics for parsers")
+ for parser_name in self.get_parser_list():
+ Logger.info("Creating topic'{}'".format(parser_name))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ parser_name,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Creating topics for error handling")
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_invalid",
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_error",
+ num_partitions, replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def init_parser_config(self):
+ Logger.info('Loading parser config into ZooKeeper')
+ Execute(format(
+ "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
+ path=format("{java_home}/bin")
+ )
+
+ def start_parser_topologies(self):
+ Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list()))
+ start_cmd_template = """{}/bin/start_parser_topology.sh \
+ -k {} \
+ -z {} \
+ -s {}"""
+ for parser in self.get_parser_list():
+ Logger.info('Starting ' + parser)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
+ self.__params.zookeeper_quorum, parser))
+
+ Logger.info('Finished starting parser topologies')
+
+ def stop_parser_topologies(self):
+ Logger.info('Stopping parsers')
+ for parser in self.get_parser_list():
+ Logger.info('Stopping ' + parser)
+ stop_cmd = 'storm kill ' + parser
+ Execute(stop_cmd)
+ Logger.info('Done stopping parser topologies')
+
+ def restart_parser_topologies(self):
+ Logger.info('Restarting the parser topologies')
+ self.stop_parser_topologies()
+ attempt_count = 0
+ while self.topologies_exist():
+ if attempt_count > 2:
+ raise Exception("Unable to kill topologies")
+ attempt_count += 1
+ time.sleep(10)
+ self.start_parser_topologies()
+ Logger.info('Done restarting the parser topologies')
+
+ def topologies_exist(self):
+ cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (stdout, stderr) = cmd_open.communicate()
+ stdout_lines = stdout.splitlines()
+ if stdout_lines:
+ status_lines = self.__get_status_lines(stdout_lines)
+ for parser in self.get_parser_list():
+ for line in status_lines:
+ items = re.sub('[\s]+', ' ', line).split()
+ if items and items[0] == parser:
+ return True
+ return False
+
+ def topologies_running(self):
+ cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (stdout, stderr) = cmd_open.communicate()
+ stdout_lines = stdout.splitlines()
+ all_running = False
+ if stdout_lines:
+ all_running = True
+ status_lines = self.__get_status_lines(stdout_lines)
+ for parser in self.get_parser_list():
+ parser_found = False
+ is_running = False
+ for line in status_lines:
+ items = re.sub('[\s]+', ' ', line).split()
+ if items and items[0] == parser:
+ status = items[1]
+ parser_found = True
+ is_running = self.__is_running(status)
+ all_running &= parser_found and is_running
+ return all_running
+
+ def __get_status_lines(self, lines):
+ status_lines = []
+ do_stat = False
+ skipped = 0
+ for line in lines:
+ if line.startswith("Topology_name"):
+ do_stat = True
+ if do_stat and skipped == 2:
+ status_lines += [line]
+ elif do_stat:
+ skipped += 1
+ return status_lines
+
+ def __is_running(self, status):
+ return status in [self.StormStatus.ACTIVE, self.StormStatus.REBALANCING]
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
new file mode 100755
index 0000000..242460e
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
@@ -0,0 +1,18 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
new file mode 100755
index 0000000..953435d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
new file mode 100755
index 0000000..de7dfc9
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import functools
+import os
+
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.script import Script
+
+# server configurations
+config = Script.get_config()
+
+hostname = config['hostname']
+#print(config['configurations'])
+metron_home = config['configurations']['metron-parsers']['metron_home']
+parsers = config['configurations']['metron-parsers']['parsers']
+metron_user = config['configurations']['metron-parsers']['metron_user']
+metron_group = config['configurations']['metron-parsers']['metron_group']
+metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir']
+#metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir
+metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
+configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured'
+yum_repo_type = 'local'
+
+# hadoop params
+stack_root = Script.get_stack_root()
+hadoop_home_dir = stack_select.get_hadoop_dir("home")
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+kafka_bin_dir = os.path.join(kafka_home, "bin")
+
+# zookeeper
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+has_zk_host = not len(zk_hosts) == 0
+zookeeper_quorum = None
+if has_zk_host:
+ if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+ else:
+ zookeeper_clientPort = '2181'
+ zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
+ # last port config
+ zookeeper_quorum += ':' + zookeeper_clientPort
+
+# Kafka
+kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
+has_kafka_host = not len(kafka_hosts) == 0
+kafka_brokers = None
+if has_kafka_host:
+ if 'port' in config['configurations']['kafka-broker']:
+ kafka_broker_port = config['configurations']['kafka-broker']['port']
+ else:
+ kafka_broker_port = '6667'
+ kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
+ kafka_brokers += ':' + kafka_broker_port
+
+metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir']
+
+local_grok_patterns_dir = format("{metron_home}/patterns")
+hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns")
+
+# for create_hdfs_directory
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hostname = config["hostname"]
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+dfs_type = default("/commandParams/dfs_type", "")
+
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled=security_enabled,
+ keytab=hdfs_user_keytab,
+ kinit_path_local=kinit_path_local,
+ hadoop_bin_dir=hadoop_bin_dir,
+ hadoop_conf_dir=hadoop_conf_dir,
+ principal_name=hdfs_principal_name,
+ hdfs_site=hdfs_site,
+ default_fs=default_fs,
+ immutable_paths=get_not_managed_resources(),
+ dfs_type=dfs_type
+)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
new file mode 100755
index 0000000..4d11b35
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
new file mode 100755
index 0000000..ee83414
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+parsers = "bro,yaf,snort"
+configured_flag_file = ""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
new file mode 100755
index 0000000..4a5b910
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
@@ -0,0 +1,77 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.libraries.script import Script
+
+from commands import Commands
+
+
+class ParserMaster(Script):
+ def get_component_name(self):
+ # TODO add this at some point - currently will cause problems with hdp-select
+ # return "parser-master"
+ pass
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = Commands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = Commands(params)
+ if not commands.is_configured():
+ commands.init_parsers()
+ commands.init_kafka_topics()
+ commands.init_parser_config()
+ commands.set_configured()
+ commands.start_parser_topologies()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = Commands(params)
+ commands.stop_parser_topologies()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = Commands(status_params)
+ if not commands.topologies_running():
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ commands = Commands(params)
+ commands.restart_parser_topologies()
+
+
+if __name__ == "__main__":
+ ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
new file mode 100755
index 0000000..6ef7e4a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+from resource_management.libraries.script import Script
+
+from commands import Commands
+
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ from params import params
+ env.set_params(params)
+ commands = Commands(params)
+ all_found = commands.topologies_running()
+ if all_found:
+ exit(0)
+ else:
+ exit(1)
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
new file mode 100755
index 0000000..0967ef4
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
@@ -0,0 +1 @@
+{}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
new file mode 100755
index 0000000..ee1b225
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "storm_ui",
+ "label": "Storm UI",
+ "requires_user_name": "false",
+ "component_name": "STORM_UI_SERVER",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "ui.port",
+ "http_default_port": "8744",
+ "https_property": "ui.port",
+ "https_default_port": "8744",
+ "regex": "^(\\d+)$",
+ "site": "storm-site"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
new file mode 100755
index 0000000..84b466a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import imp
+import os
+import traceback
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE))
+
+
+class TODO_PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor):
+ # colocate Metron Parser Master with KAFKA_BROKERs
+ def TODO_colocateService(self, hostsComponentsMap, serviceComponents):
+ parsersMasterComponent = [component for component in serviceComponents if
+ component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"][0]
+ if not self.isComponentHostsPopulated(parsersMasterComponent):
+ for hostName in hostsComponentsMap.keys():
+ hostComponents = hostsComponentsMap[hostName]
+ if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "PARSER_MASTER"} not in hostComponents:
+ hostsComponentsMap[hostName].append({"name": "PARSER_MASTER"})
+ if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "PARSER_MASTER"} in hostComponents:
+ hostsComponentsMap[hostName].remove({"name": "PARSER_MASTER"})
+
+ def TODO_getServiceComponentLayoutValidations(self, services, hosts):
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+ hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ hostsCount = len(hostsList)
+
+ masterHosts = self.getHosts(componentsList, "PARSER_MASTER")
+ expectedMasterHosts = set(self.getHosts(componentsList, "KAFKA_BROKER"))
+
+ items = []
+
+ mismatchHosts = sorted(expectedMasterHosts.symmetric_difference(set(masterHosts)))
+ if len(mismatchHosts) > 0:
+ hostsString = ', '.join(mismatchHosts)
+ message = "Metron Parsers Master must be installed on Kafka Brokers. " \
+ "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format(
+ len(mismatchHosts), hostsString)
+ items.append(
+ {"type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PARSER_MASTER'})
+
+ return items
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
new file mode 100755
index 0000000..e0292cf
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>PARSERS</name>
+ <version>0.2.0BETA</version>
+ <extends>common-services/PARSERS/0.2.0BETA</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
new file mode 100755
index 0000000..c861646
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
@@ -0,0 +1,17 @@
+{
+ "_comment": "Record format:",
+ "_comment": "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps": {
+ "_comment": "dependencies for Metron Parsers",
+ "PARSER_MASTER-START": [
+ "NAMENODE-START",
+ "ZOOKEEPER_SERVER-START",
+ "KAFKA_BROKER-START",
+ "STORM_REST_API-START"
+ ],
+ "_comment": "<SERVICE_NAME>_SERVICE_CHECK-SERVICE_CHECK",
+ "PARSERS_SERVICE_CHECK-SERVICE_CHECK": [
+ "PARSER_MASTER-START"
+ ]
+ }
+}