You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2017/09/15 15:42:37 UTC
metron git commit: METRON-1187 Indexing/Profiler Kafka ACL Groups Not
Setup Correctly (nickwallen) closes apache/metron#759
Repository: metron
Updated Branches:
refs/heads/master 9b7bf7cd7 -> c642a9558
METRON-1187 Indexing/Profiler Kafka ACL Groups Not Setup Correctly (nickwallen) closes apache/metron#759
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c642a955
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c642a955
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c642a955
Branch: refs/heads/master
Commit: c642a9558f806b3a49c6cca1aefc4b32799a9a4d
Parents: 9b7bf7c
Author: nickwallen <ni...@nickallen.org>
Authored: Fri Sep 15 11:42:05 2017 -0400
Committer: nickallen <ni...@apache.org>
Committed: Fri Sep 15 11:42:05 2017 -0400
----------------------------------------------------------------------
.../package/scripts/enrichment_commands.py | 20 ++++++-------
.../package/scripts/enrichment_master.py | 1 -
.../package/scripts/indexing_commands.py | 30 +++++++++++---------
.../CURRENT/package/scripts/metron_service.py | 6 ++--
.../package/scripts/params/params_linux.py | 4 +--
.../CURRENT/package/scripts/parser_commands.py | 14 ++++-----
.../package/scripts/profiler_commands.py | 23 +++++++--------
7 files changed, 48 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/c642a955/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index f9a75b1..1df448b 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -53,14 +53,14 @@ class EnrichmentCommands:
return self.__kafka_acl_configured
def set_kafka_configured(self):
- Logger.info("Setting Kafka Configured to True")
+ Logger.info("Setting Kafka Configured to True for enrichment")
File(self.__params.enrichment_kafka_configured_flag_file,
content="",
owner=self.__params.metron_user,
mode=0755)
def set_kafka_acl_configured(self):
- Logger.info("Setting Kafka ACL Configured to True")
+ Logger.info("Setting Kafka ACL Configured to True for enrichment")
File(self.__params.enrichment_kafka_acl_configured_flag_file,
content="",
owner=self.__params.metron_user,
@@ -73,14 +73,14 @@ class EnrichmentCommands:
return self.__hbase_acl_configured
def set_hbase_configured(self):
- Logger.info("Setting HBase Configured to True")
+ Logger.info("Setting HBase Configured to True for enrichment")
File(self.__params.enrichment_hbase_configured_flag_file,
content="",
owner=self.__params.metron_user,
mode=0755)
def set_hbase_acl_configured(self):
- Logger.info("Setting HBase ACL Configured to True")
+ Logger.info("Setting HBase ACL Configured to True for enrichment")
File(self.__params.enrichment_hbase_acl_configured_flag_file,
content="",
owner=self.__params.metron_user,
@@ -90,7 +90,7 @@ class EnrichmentCommands:
return self.__geo_configured
def set_geo_configured(self):
- Logger.info("Setting GEO Configured to True")
+ Logger.info("Setting GEO Configured to True for enrichment")
File(self.__params.enrichment_geo_configured_flag_file,
content="",
owner=self.__params.metron_user,
@@ -144,10 +144,10 @@ class EnrichmentCommands:
-s {1} \
-z {2}"""
Logger.info('Starting ' + self.__enrichment_topology)
- Execute(start_cmd_template.format(self.__params.metron_home,
- self.__enrichment_topology,
- self.__params.zookeeper_quorum),
- user=self.__params.metron_user)
+ start_cmd = start_cmd_template.format(self.__params.metron_home,
+ self.__enrichment_topology,
+ self.__params.zookeeper_quorum)
+ Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
else:
Logger.info('Enrichment topology already running')
@@ -158,7 +158,7 @@ class EnrichmentCommands:
if self.is_topology_active(env):
stop_cmd = 'storm kill ' + self.__enrichment_topology
- Execute(stop_cmd, user=self.__params.metron_user)
+ Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
else:
Logger.info("Enrichment topology already stopped")
http://git-wip-us.apache.org/repos/asf/metron/blob/c642a955/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
index f3453ea..a72f69e 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
@@ -98,7 +98,6 @@ class Enrichment(Script):
status_params.metron_principal_name,
execute_user=status_params.metron_user)
-
if not commands.is_topology_active(env):
raise ComponentIsNotRunning()
http://git-wip-us.apache.org/repos/asf/metron/blob/c642a955/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 901ab83..8dde568 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -46,6 +46,7 @@ class IndexingCommands:
self.__acl_configured = os.path.isfile(self.__params.indexing_acl_configured_flag_file)
self.__hbase_configured = os.path.isfile(self.__params.indexing_hbase_configured_flag_file)
self.__hbase_acl_configured = os.path.isfile(self.__params.indexing_hbase_acl_configured_flag_file)
+ self.__hdfs_perm_configured = os.path.isfile(self.__params.indexing_hdfs_perm_configured_flag_file)
def is_configured(self):
return self.__configured
@@ -69,21 +70,21 @@ class IndexingCommands:
return self.__hbase_acl_configured
def set_hbase_configured(self):
- Logger.info("Setting HBase Configured to True")
+ Logger.info("Setting HBase Configured to True for indexing")
File(self.__params.indexing_hbase_configured_flag_file,
content="",
owner=self.__params.metron_user,
mode=0755)
def set_hbase_acl_configured(self):
- Logger.info("Setting HBase ACL Configured to True")
+ Logger.info("Setting HBase ACL Configured to True for indexing")
File(self.__params.indexing_hbase_acl_configured_flag_file,
content="",
owner=self.__params.metron_user,
mode=0755)
def create_hbase_tables(self):
- Logger.info("Creating HBase Tables")
+ Logger.info("Creating HBase Tables for indexing")
if self.__params.security_enabled:
metron_security.kinit(self.__params.kinit_path_local,
self.__params.hbase_keytab_path,
@@ -99,11 +100,11 @@ class IndexingCommands:
user=self.__params.hbase_user
)
- Logger.info("Done creating HBase Tables")
+ Logger.info("Done creating HBase Tables for indexing")
self.set_hbase_configured()
def set_hbase_acls(self):
- Logger.info("Setting HBase ACLs")
+ Logger.info("Setting HBase ACLs for indexing")
if self.__params.security_enabled:
metron_security.kinit(self.__params.kinit_path_local,
self.__params.hbase_keytab_path,
@@ -119,7 +120,7 @@ class IndexingCommands:
user=self.__params.hbase_user
)
- Logger.info("Done setting HBase ACLs")
+ Logger.info("Done setting HBase ACLs for indexing")
self.set_hbase_acl_configured()
def set_acl_configured(self):
@@ -139,7 +140,7 @@ class IndexingCommands:
metron_service.init_kafka_topics(self.__params, [self.__indexing_topic])
def init_kafka_acls(self):
- Logger.info('Creating Kafka ACLs')
+ Logger.info('Creating Kafka ACLs for indexing')
# Indexed topic names matches the group
metron_service.init_kafka_acls(self.__params, [self.__indexing_topic], [self.__indexing_topic])
@@ -168,13 +169,14 @@ class IndexingCommands:
self.__params.metron_keytab_path,
self.__params.metron_principal_name,
execute_user=self.__params.metron_user)
+
start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \
- -s {1} \
- -z {2}"""
- Execute(start_cmd_template.format(self.__params.metron_home,
- self.__indexing_topology,
- self.__params.zookeeper_quorum),
- user=self.__params.metron_user)
+ -s {1} \
+ -z {2}"""
+ start_cmd = start_cmd_template.format(self.__params.metron_home,
+ self.__indexing_topology,
+ self.__params.zookeeper_quorum)
+ Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
else:
Logger.info('Indexing topology already running')
@@ -191,7 +193,7 @@ class IndexingCommands:
self.__params.metron_principal_name,
execute_user=self.__params.metron_user)
stop_cmd = 'storm kill ' + self.__indexing_topology
- Execute(stop_cmd, user=self.__params.metron_user)
+ Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
else:
Logger.info("Indexing topology already stopped")
http://git-wip-us.apache.org/repos/asf/metron/blob/c642a955/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index d808110..cf278f2 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -122,7 +122,7 @@ def init_kafka_topics(params, topics):
num_partitions,
replication_factor,
retention_bytes),
- user=params.kafka_user)
+ user=params.kafka_user, tries=3, try_sleep=5, logoutput=True)
Logger.info("Done creating Kafka topics")
@@ -142,7 +142,7 @@ def init_kafka_acls(params, topics, groups):
params.zookeeper_quorum,
params.metron_user,
topic),
- user=params.kafka_user)
+ user=params.kafka_user, tries=3, try_sleep=5, logoutput=True)
acl_template = """{0}/kafka-acls.sh \
--authorizer kafka.security.auth.SimpleAclAuthorizer \
@@ -157,5 +157,5 @@ def init_kafka_acls(params, topics, groups):
params.zookeeper_quorum,
params.metron_user,
group),
- user=params.kafka_user)
+ user=params.kafka_user, tries=3, try_sleep=5, logoutput=True)
Logger.info("Done creating Kafka ACLs")
http://git-wip-us.apache.org/repos/asf/metron/blob/c642a955/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 72f295b..b273bd6 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -259,7 +259,6 @@ threat_intel_join_parallelism = config['configurations']['metron-enrichment-env'
kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['kafka_writer_parallelism']
# Profiler
-
metron_profiler_topology = 'profiler'
profiler_input_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic']
profiler_kafka_start = config['configurations']['metron-profiler-env']['profiler_kafka_start']
@@ -274,7 +273,7 @@ profiler_acker_executors = config['configurations']['metron-profiler-env']['prof
profiler_hbase_table = config['configurations']['metron-profiler-env']['profiler_hbase_table']
profiler_hbase_cf = config['configurations']['metron-profiler-env']['profiler_hbase_cf']
profiler_configured_flag_file = status_params.profiler_configured_flag_file
-profiler_acl_configured_flag_file = status_params.indexing_acl_configured_flag_file
+profiler_acl_configured_flag_file = status_params.profiler_acl_configured_flag_file
profiler_hbase_configured_flag_file = status_params.profiler_hbase_configured_flag_file
profiler_hbase_acl_configured_flag_file = status_params.profiler_hbase_acl_configured_flag_file
if not len(profiler_topology_worker_childopts) == 0:
@@ -304,4 +303,3 @@ metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-in
bolt_hdfs_rotation_policy = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy']
bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_units']
bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
-
http://git-wip-us.apache.org/repos/asf/metron/blob/c642a955/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
index 334dc55..ccad5d5 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
@@ -124,12 +124,12 @@ class ParserCommands:
for parser in stopped_parsers:
Logger.info('Starting ' + parser)
- Execute(start_cmd_template.format(self.__params.metron_home,
- self.__params.kafka_brokers,
- self.__params.zookeeper_quorum,
- parser,
- self.__params.kafka_security_protocol),
- user=self.__params.metron_user)
+ start_cmd = start_cmd_template.format(self.__params.metron_home,
+ self.__params.kafka_brokers,
+ self.__params.zookeeper_quorum,
+ parser,
+ self.__params.kafka_security_protocol)
+ Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
Logger.info('Finished starting parser topologies')
@@ -147,7 +147,7 @@ class ParserCommands:
self.__params.metron_keytab_path,
self.__params.metron_principal_name,
execute_user=self.__params.metron_user)
- Execute(stop_cmd, user=self.__params.metron_user)
+ Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
Logger.info('Done stopping parser topologies')
def restart_parser_topologies(self, env):
http://git-wip-us.apache.org/repos/asf/metron/blob/c642a955/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
index 4dfa371..5c31896 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
@@ -65,21 +65,21 @@ class ProfilerCommands:
return self.__hbase_acl_configured
def set_hbase_configured(self):
- Logger.info("Setting HBase Configured to True")
+ Logger.info("Setting HBase Configured to True for profiler")
File(self.__params.profiler_hbase_configured_flag_file,
content="",
owner=self.__params.metron_user,
mode=0755)
def set_hbase_acl_configured(self):
- Logger.info("Setting HBase ACL Configured to True")
+ Logger.info("Setting HBase ACL Configured to True for profiler")
File(self.__params.profiler_hbase_acl_configured_flag_file,
content="",
owner=self.__params.metron_user,
mode=0755)
def create_hbase_tables(self):
- Logger.info("Creating HBase Tables")
+ Logger.info("Creating HBase Tables for profiler")
if self.__params.security_enabled:
metron_security.kinit(self.__params.kinit_path_local,
self.__params.hbase_keytab_path,
@@ -95,7 +95,7 @@ class ProfilerCommands:
user=self.__params.hbase_user
)
- Logger.info("Done creating HBase Tables")
+ Logger.info("Done creating HBase Tables for profiler")
self.set_hbase_configured()
def init_kafka_acls(self):
@@ -103,7 +103,7 @@ class ProfilerCommands:
metron_service.init_kafka_acls(self.__params, [self.__profiler_topic], ['profiler'])
def set_hbase_acls(self):
- Logger.info("Setting HBase ACLs")
+ Logger.info("Setting HBase ACLs for profiler")
if self.__params.security_enabled:
metron_security.kinit(self.__params.kinit_path_local,
self.__params.hbase_keytab_path,
@@ -119,7 +119,7 @@ class ProfilerCommands:
user=self.__params.hbase_user
)
- Logger.info("Done setting HBase ACLs")
+ Logger.info("Done setting HBase ACLs for profiler")
self.set_hbase_acl_configured()
def set_acl_configured(self):
@@ -140,11 +140,10 @@ class ProfilerCommands:
start_cmd_template = """{0}/bin/start_profiler_topology.sh \
-s {1} \
-z {2}"""
- Execute(start_cmd_template.format(self.__params.metron_home,
- self.__profiler_topology,
- self.__params.zookeeper_quorum),
- user=self.__params.metron_user)
-
+ start_cmd = start_cmd_template.format(self.__params.metron_home,
+ self.__profiler_topology,
+ self.__params.zookeeper_quorum)
+ Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
else:
Logger.info('Profiler topology already running')
@@ -160,7 +159,7 @@ class ProfilerCommands:
self.__params.metron_principal_name,
execute_user=self.__params.metron_user)
stop_cmd = 'storm kill ' + self.__profiler_topology
- Execute(stop_cmd, user=self.__params.metron_user)
+ Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
else:
Logger.info("Profiler topology already stopped")