You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2018/09/20 15:10:23 UTC
metron git commit: METRON-1709 Add controls to start / stop the PCAP
topology from Ambari. (MohanDV via nickwallen) closes apache/metron#1201
Repository: metron
Updated Branches:
refs/heads/master 3372aa360 -> 15194c3b4
METRON-1709 Add controls to start / stop the PCAP topology from Ambari. (MohanDV via nickwallen) closes apache/metron#1201
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/15194c3b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/15194c3b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/15194c3b
Branch: refs/heads/master
Commit: 15194c3b4e7d9d9d161b0ddfa556395b34cfef7c
Parents: 3372aa3
Author: MohanDV <mo...@gmail.com>
Authored: Thu Sep 20 11:10:07 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Thu Sep 20 11:10:07 2018 -0400
----------------------------------------------------------------------
.../roles/ambari_config/vars/single_node_vm.yml | 3 +-
.../roles/ambari_config/vars/small_cluster.yml | 3 +-
.../METRON/CURRENT/role_command_order.json | 9 +-
.../common-services/METRON/CURRENT/metainfo.xml | 39 ++++
.../package/scripts/params/params_linux.py | 4 +
.../package/scripts/params/status_params.py | 1 +
.../CURRENT/package/scripts/pcap_commands.py | 201 +++++++++++++++++++
.../CURRENT/package/scripts/pcap_master.py | 105 ++++++++++
.../CURRENT/package/scripts/rest_commands.py | 43 ----
.../CURRENT/package/scripts/rest_master.py | 7 -
.../METRON/CURRENT/service_advisor.py | 5 +
11 files changed, 365 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index a3c643b..962726c 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -32,6 +32,7 @@ es_master: [ES_MASTER]
kibana_master: [KIBANA_MASTER]
metron_indexing: [METRON_INDEXING]
metron_profiler: [METRON_PROFILER]
+metron_pcap: [METRON_PCAP]
metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
metron_parsers : [METRON_PARSERS]
metron_rest: [METRON_REST]
@@ -40,7 +41,7 @@ metron_alerts_ui: [METRON_ALERTS_UI]
metron_components: >
{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) |
- union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | union(metron_profiler) |
+ union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | union(metron_profiler) | union(metron_pcap) |
union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(es_master) }}
cluster_name: "metron_cluster"
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
index 218e267..818b5f3 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
@@ -32,6 +32,7 @@ es_slave: [ES_SLAVE]
kibana_master: [KIBANA_MASTER]
metron_indexing: [METRON_INDEXING]
metron_profiler: [METRON_PROFILER]
+metron_pcap: [METRON_PCAP]
metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
metron_parsers : [METRON_PARSERS]
metron_rest: [METRON_REST]
@@ -45,7 +46,7 @@ master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_ma
master_2_host:
- "{{groups.ambari_slave[1]}}"
metron_components: >
- {{ metron_indexing | union(metron_profiler) | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) | union(storm_slave) |
+ {{ metron_indexing | union(metron_profiler) | union(metron_pcap) |union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) | union(storm_slave) |
union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
metron_host:
- "{{ groups.metron[0] }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
index fcb2206..ee6ca23 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
@@ -6,23 +6,26 @@
"METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
"METRON_ENRICHMENT-INSTALL": ["METRON_INDEXING-INSTALL"],
"METRON_PROFILER-INSTALL" : ["METRON_ENRICHMENT-INSTALL"],
+ "METRON_PCAP-INSTALL" : ["METRON_INDEXING-INSTALL"],
"METRON_REST-INSTALL" : ["METRON_PARSERS-INSTALL"],
"METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START" ,"METRON_ENRICHMENT_MASTER-START"],
"METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
"METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "METRON_PARSERS-START"],
"METRON_PROFILER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START", "METRON_ENRICHMENT-INSTALL"],
+ "METRON_PCAP-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
"METRON_REST-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "METRON_PARSERS-INSTALL","METRON_INDEXING-INSTALL","METRON_ENRICHMENT-INSTALL"],
"METRON_MANAGEMENT_UI-START" : ["METRON_REST-START"],
"METRON_ALERTS_UI-START" : ["METRON_REST-START"],
- "STORM_REST_API-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP", "METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP", "METRON_ALERTS_UI-STOP"],
- "STORM_UI_SERVER-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP", "METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP", "METRON_ALERTS_UI-STOP"],
+ "STORM_REST_API-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP", "METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_PCAP-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP", "METRON_ALERTS_UI-STOP"],
+ "STORM_UI_SERVER-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP", "METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_PCAP-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP", "METRON_ALERTS_UI-STOP"],
- "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START", "METRON_ENRICHMENT_MASTER-START", "METRON_INDEXING-START", "METRON_PROFILER-START", "METRON_REST-START", "METRON_MANAGEMENT_UI-START", "METRON_ALERTS_UI-START"],
+ "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START", "METRON_ENRICHMENT_MASTER-START", "METRON_INDEXING-START", "METRON_PROFILER-START", "METRON_PCAP-START", "METRON_REST-START", "METRON_MANAGEMENT_UI-START", "METRON_ALERTS_UI-START"],
"METRON_PARSERS_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_ENRICHMENT_MASTER_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK", "HBASE_MASTER_SERVICE_CHECK-SERVICE_CHECK", "HBASE_REGIONSERVER_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_INDEXING_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_INDEXING-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_PROFILER_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PROFILER-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK", "HBASE_MASTER_SERVICE_CHECK-SERVICE_CHECK", "HBASE_REGIONSERVER_SERVICE_CHECK-SERVICE_CHECK"],
+ "METRON_PCAP_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PCAP-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_REST_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_REST-START", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_MANAGEMENT_UI_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_MANAGEMENT_UI-START", "METRON_REST_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_ALERTS_UI_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ALERTS_UI-START", "METRON_REST_SERVICE_CHECK-SERVICE_CHECK"]
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 644ba97..5965506 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -159,6 +159,45 @@
</component>
<component>
+ <name>METRON_PCAP</name>
+ <displayName>Metron PCAP</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/pcap_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <configuration-dependencies>
+ <config-type>metron-rest-env</config-type>
+ </configuration-dependencies>
+ </component>
+
+ <component>
<name>METRON_INDEXING</name>
<displayName>Metron Indexing</displayName>
<category>MASTER</category>
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 0525c7f..dbad44d 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -388,6 +388,8 @@ bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env'
bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
# Pcap
+metron_pcap_topology = 'pcap'
+pcap_input_topic = 'pcap'
pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path']
@@ -396,6 +398,8 @@ pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue']
pcap_finalizer_threadpool_size= config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size']
pcap_configured_flag_file = status_params.pcap_configured_flag_file
pcap_perm_configured_flag_file = status_params.pcap_perm_configured_flag_file
+pcap_acl_configured_flag_file = status_params.pcap_acl_configured_flag_file
+
# MapReduce
metron_user_hdfs_dir = '/user/' + metron_user
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 99f5ec0..2c711cf 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -121,6 +121,7 @@ metron_keytab_path = config['configurations']['metron-env']['metron_service_keyt
# Pcap
pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured'
pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured'
+pcap_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_acl_configured'
# MapReduce
metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + '/../metron_user_hdfs_dir_configured'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py
new file mode 100644
index 0000000..2190916
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py
@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import subprocess
+import time
+
+from datetime import datetime
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+import metron_security
+
+
+# Wrap major operations and functionality in this class
+class PcapCommands:
+ __params = None
+ __configured = False
+ __acl_configured = False
+ __pcap_topology = None
+ __pcap_perm_configured = False
+ __pcap_topic = None
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__configured = os.path.isfile(self.__params.pcap_configured_flag_file)
+ self.__acl_configured = os.path.isfile(self.__params.pcap_acl_configured_flag_file)
+ self.__pcap_perm_configured = os.path.isfile(self.__params.pcap_perm_configured_flag_file)
+ self.__pcap_topology = params.metron_pcap_topology
+ self.__pcap_topic = params.pcap_input_topic
+
+ def __get_topics(self):
+ return [self.__pcap_topic]
+
+ def __get_kafka_acl_groups(self):
+ return ['pcap']
+
+ def is_configured(self):
+ return self.__configured
+
+ def is_acl_configured(self):
+ return self.__acl_configured
+
+ def is_hdfs_perm_configured(self):
+ return self.__pcap_perm_configured
+
+ def set_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.pcap_configured_flag_file, "Setting PCAP configured to True")
+
+ def set_acl_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.pcap_acl_configured_flag_file, "Setting PCAP ACL configured to true")
+
+ def set_hdfs_perm_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.pcap_perm_configured_flag_file, "Setting PCAP HDFS perm configured to True")
+
+ def init_pcap(self):
+ self.init_kafka_topics()
+ self.init_hdfs_dir()
+ Logger.info("Done initializing PCAP configuration")
+
+ def init_hdfs_dir(self):
+ Logger.info("Creating HDFS locations for PCAP")
+ # Non Kerberized Metron runs under 'storm', requiring write under the 'hadoop' group.
+ # Kerberized Metron runs under it's own user.
+ ownership = 0755 if self.__params.security_enabled else 0775
+ self.__params.HdfsResource(self.__params.pcap_base_path,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.hadoop_group,
+ mode=ownership,
+ )
+ self.__params.HdfsResource(self.__params.pcap_base_interim_result_path,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.hadoop_group,
+ mode=ownership,
+ )
+ self.__params.HdfsResource(self.__params.pcap_final_output_path,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.hadoop_group,
+ mode=ownership,
+ )
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topic for PCAP')
+ metron_service.init_kafka_topics(self.__params, self.__get_topics())
+
+ def init_kafka_acls(self):
+ Logger.info('Creating Kafka ACLs for PCAP')
+ metron_service.init_kafka_acls(self.__params, self.__get_topics())
+ metron_service.init_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups())
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+ active = True
+ topologies = metron_service.get_running_topologies(self.__params)
+ is_running = False
+ if self.__pcap_topology in topologies:
+ is_running = topologies[self.__pcap_topology] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
+
+ def start_pcap_topology(self, env):
+ Logger.info('Starting Metron PCAP topology')
+ start_cmd_template = """{0}/bin/start_pcap_topology.sh"""
+ if not self.is_topology_active(env):
+ if self.__params.security_enabled:
+ metron_security.kinit(self.__params.kinit_path_local,
+ self.__params.metron_keytab_path,
+ self.__params.metron_principal_name,
+ execute_user=self.__params.metron_user)
+ start_cmd = start_cmd_template.format(self.__params.metron_home)
+ Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
+ else :
+ Logger.info('PCAP topology already started')
+
+ Logger.info('Finished starting pcap topologies')
+
+ def stop_pcap_topology(self, env):
+ Logger.info('Stopping Metron PCAP topology')
+ if self.is_topology_active(env):
+ if self.__params.security_enabled:
+ metron_security.kinit(self.__params.kinit_path_local,
+ self.__params.metron_keytab_path,
+ self.__params.metron_principal_name,
+ execute_user=self.__params.metron_user)
+ stop_cmd = 'storm kill ' + self.__pcap_topology
+ Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
+
+ else :
+ Logger.info('PCAP topology already stopped')
+
+ Logger.info('Finished starting PCAP topologies')
+
+ def restart_pcap_topology(self, env):
+ Logger.info('Restarting the PCAP topology')
+ self.stop_pcap_topology(env)
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while self.is_topology_active(env) and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(10)
+ retries += 1
+
+ if not topology_active:
+ Logger.info('Waiting for storm kill to complete')
+ time.sleep(30)
+ self.start_pcap_topology(env)
+ Logger.info('Done restarting the PCAP topologies')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def service_check(self, env):
+ """
+ Performs a service check for the PCAP.
+ :param env: Environment
+ """
+ Logger.info('Checking Kafka topic for PCAP')
+ metron_service.check_kafka_topics(self.__params, self.__get_topics())
+
+ Logger.info("Checking for PCAP sequence files directory in HDFS for PCAP")
+ metron_service.check_hdfs_dir_exists(self.__params, self.__params.hdfs_pcap_sequencefiles_dir)
+
+ if self.__params.security_enabled:
+ Logger.info('Checking Kafka ACLs for PCAP')
+ metron_service.check_kafka_acls(self.__params, self.__get_topics())
+ metron_service.check_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups())
+
+ Logger.info("Checking for PCAP topologies")
+ if not self.is_topology_active(env):
+ raise Fail("PCAP topologies not running")
+
+ Logger.info("PCAP service check completed successfully")
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
new file mode 100644
index 0000000..a3bc1b4
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
@@ -0,0 +1,105 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import Template
+from resource_management.libraries.functions.format import format
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.script import Script
+
+from metron_security import storm_security_setup
+import metron_service
+import metron_security
+from pcap_commands import PcapCommands
+
+
+class Pcap(Script):
+ __configured = False
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ if not metron_service.is_zk_configured(params):
+ metron_service.init_zk_config(params)
+ metron_service.set_zk_configured(params)
+ metron_service.refresh_configs(params)
+
+ commands = PcapCommands(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.init_hdfs_dir()
+ if params.security_enabled and not commands.is_acl_configured():
+ commands.init_kafka_acls()
+ commands.set_acl_configured()
+ Logger.info("Calling security setup")
+ storm_security_setup(params)
+ if not commands.is_configured():
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = PcapCommands(params)
+ if params.security_enabled:
+ metron_security.kinit(params.kinit_path_local,
+ params.metron_keytab_path,
+ params.metron_principal_name,
+ execute_user=params.metron_user)
+
+ if params.security_enabled and not commands.is_hdfs_perm_configured():
+ commands.init_hdfs_dir()
+ commands.set_hdfs_perm_configured()
+ if params.security_enabled and not commands.is_acl_configured():
+ commands.init_kafka_acls()
+ commands.set_acl_configured()
+
+ commands.start_pcap_topology(env)
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = PcapCommands(params)
+ commands.stop_pcap_topology(env)
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = PcapCommands(status_params)
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = PcapCommands(params)
+ commands.restart_pcap_topology(env)
+
+if __name__ == "__main__":
+ Pcap().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index d44f478..c410b94 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -37,8 +37,6 @@ class RestCommands:
__kafka_acl_configured = False
__hbase_configured = False
__hbase_acl_configured = False
- __pcap_configured = False
- __pcap_perm_configured = False
__metron_user_hdfs_dir_configured = False
def __init__(self, params):
@@ -49,8 +47,6 @@ class RestCommands:
self.__kafka_acl_configured = os.path.isfile(self.__params.rest_kafka_acl_configured_flag_file)
self.__hbase_configured = os.path.isfile(self.__params.rest_hbase_configured_flag_file)
self.__hbase_acl_configured = os.path.isfile(self.__params.rest_hbase_acl_configured_flag_file)
- self.__pcap_configured = os.path.isfile(self.__params.pcap_configured_flag_file)
- self.__pcap_perm_configured = os.path.isfile(self.__params.pcap_perm_configured_flag_file)
self.__metron_user_hdfs_dir_configured = os.path.isfile(self.__params.metron_user_hdfs_dir_configured_flag_file)
Directory(params.metron_rest_pid_dir,
mode=0755,
@@ -80,12 +76,6 @@ class RestCommands:
def is_hbase_acl_configured(self):
return self.__hbase_acl_configured
- def is_pcap_configured(self):
- return self.__pcap_configured
-
- def is_pcap_perm_configured(self):
- return self.__pcap_perm_configured
-
def is_metron_user_hdfs_dir_configured(self):
return self.__metron_user_hdfs_dir_configured
@@ -101,12 +91,6 @@ class RestCommands:
def set_hbase_acl_configured(self):
metron_service.set_configured(self.__params.metron_user, self.__params.rest_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for rest")
- def set_pcap_configured(self):
- metron_service.set_configured(self.__params.metron_user, self.__params.pcap_configured_flag_file, "Setting Pcap configured to True")
-
- def set_pcap_perm_configured(self):
- metron_service.set_configured(self.__params.metron_user, self.__params.pcap_perm_configured_flag_file, "Setting Pcap perm configured to True")
-
def set_metron_user_hdfs_dir_configured(self):
metron_service.set_configured(self.__params.metron_user, self.__params.metron_user_hdfs_dir_configured_flag_file, "Setting Metron user HDFS directory configured to True")
@@ -124,33 +108,6 @@ class RestCommands:
groups = ['metron-rest']
metron_service.init_kafka_acl_groups(self.__params, groups)
- def init_pcap(self):
- Logger.info("Creating HDFS locations for Pcap")
- # Non Kerberized Metron runs under 'storm', requiring write under the 'hadoop' group.
- # Kerberized Metron runs under it's own user.
- ownership = 0755 if self.__params.security_enabled else 0775
- self.__params.HdfsResource(self.__params.pcap_base_path,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- group=self.__params.hadoop_group,
- mode=ownership,
- )
- self.__params.HdfsResource(self.__params.pcap_base_interim_result_path,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- group=self.__params.hadoop_group,
- mode=ownership,
- )
- self.__params.HdfsResource(self.__params.pcap_final_output_path,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- group=self.__params.hadoop_group,
- mode=ownership,
- )
-
def create_metron_user_hdfs_dir(self):
Logger.info("Creating HDFS location for Metron user")
self.__params.HdfsResource(self.__params.metron_user_hdfs_dir,
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
index 791ca77..43224ad 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
@@ -51,8 +51,6 @@ class RestMaster(Script):
commands.init_kafka_topics()
if not commands.is_hbase_configured():
commands.create_hbase_tables()
- if not commands.is_pcap_configured():
- commands.init_pcap()
if not commands.is_metron_user_hdfs_dir_configured():
commands.create_metron_user_hdfs_dir()
if params.security_enabled and not commands.is_hbase_acl_configured():
@@ -60,11 +58,6 @@ class RestMaster(Script):
if params.security_enabled and not commands.is_kafka_acl_configured():
commands.init_kafka_acls()
commands.set_kafka_acl_configured()
- if params.security_enabled and not commands.is_pcap_perm_configured():
- # If we Kerberize the cluster, we need to call this again, to remove write perms from hadoop group
- # If we start off Kerberized, it just does the same thing twice.
- commands.init_pcap()
- commands.set_pcap_perm_configured()
def start(self, env, upgrade_type=None):
from params import params
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
index 3bf5c7c..b008fc0 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
@@ -44,6 +44,7 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0]
metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0]
metronProfilerHost = self.getHosts(componentsList, "METRON_PROFILER")[0]
+ metronPcapHost = self.getHosts(componentsList, "METRON_PCAP")[0]
metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0]
metronRESTHost = self.getHosts(componentsList, "METRON_REST")[0]
metronManagementUIHost = self.getHosts(componentsList, "METRON_MANAGEMENT_UI")[0]
@@ -83,6 +84,10 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
message = "Metron Indexing must be co-located with Metron Parsers on {0}".format(metronParsersHost)
items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_INDEXING', "host": metronIndexingHost })
+ if metronParsersHost != metronPcapHost:
+ message = "Metron PCAP must be co-located with Metron Parsers on {0}".format(metronParsersHost)
+ items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_PCAP', "host": metronPcapHost })
+
if metronParsersHost != metronProfilerHost:
message = "Metron Profiler must be co-located with Metron Parsers on {0}".format(metronParsersHost)
items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_PROFILER', "host": metronProfilerHost })