You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2018/01/30 14:54:11 UTC
[29/31] metron git commit: METRON-1410 Some more upgrade fallout...
Can' t restart Metron Indexing. (ottobackwards) closes apache/metron#901
METRON-1410 Some more upgrade fallout... Can't restart Metron Indexing. (ottobackwards) closes apache/metron#901
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0c6aad8c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0c6aad8c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0c6aad8c
Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 0c6aad8c7c7affd6c898e1dd7663833a71ba63de
Parents: 6f26799
Author: ottobackwards <ot...@gmail.com>
Authored: Sat Jan 27 17:22:46 2018 -0500
Committer: otto <ot...@apache.org>
Committed: Sat Jan 27 17:22:46 2018 -0500
----------------------------------------------------------------------
.../package/scripts/enrichment_commands.py | 36 +++++---------------
.../package/scripts/indexing_commands.py | 18 ++--------
.../CURRENT/package/scripts/metron_service.py | 26 ++++++++++++++
.../package/scripts/profiler_commands.py | 26 ++++----------
4 files changed, 45 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index 90a690e..f9ec547 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -17,14 +17,15 @@ limitations under the License.
import os
import time
-from datetime import datetime
+
from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
+from resource_management.core.resources.system import Execute
import metron_service
from metron_security import kinit
+
# Wrap major operations and functionality in this class
class EnrichmentCommands:
__params = None
@@ -184,31 +185,12 @@ class EnrichmentCommands:
def create_hbase_tables(self):
Logger.info("Creating HBase Tables")
- if self.__params.security_enabled:
- kinit(self.__params.kinit_path_local,
- self.__params.hbase_keytab_path,
- self.__params.hbase_principal_name,
- execute_user=self.__params.hbase_user)
-
- cmd = "echo \"create '{0}','{1}'\" | hbase shell -n"
- add_enrichment_cmd = cmd.format(self.__params.enrichment_hbase_table, self.__params.enrichment_hbase_cf)
- Execute(add_enrichment_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- user=self.__params.hbase_user
- )
-
- add_threatintel_cmd = cmd.format(self.__params.threatintel_hbase_table, self.__params.threatintel_hbase_cf)
- Execute(add_threatintel_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- user=self.__params.hbase_user
- )
-
+ metron_service.create_hbase_table(self.__params,
+ self.__params.enrichment_hbase_table,
+ self.__params.enrichment_hbase_cf)
+ metron_service.create_hbase_table(self.__params,
+ self.__params.threatintel_hbase_table,
+ self.__params.threatintel_hbase_cf)
Logger.info("Done creating HBase Tables")
self.set_hbase_configured()
http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 33f45d4..c057b72 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -113,21 +113,9 @@ class IndexingCommands:
def create_hbase_tables(self):
Logger.info("Creating HBase Tables for indexing")
- if self.__params.security_enabled:
- metron_security.kinit(self.__params.kinit_path_local,
- self.__params.hbase_keytab_path,
- self.__params.hbase_principal_name,
- execute_user=self.__params.hbase_user)
- cmd = "echo \"create '{0}','{1}'\" | hbase shell -n"
- add_update_cmd = cmd.format(self.__params.update_hbase_table, self.__params.update_hbase_cf)
- Execute(add_update_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- user=self.__params.hbase_user
- )
-
+ metron_service.create_hbase_table(self.__params,
+ self.__params.update_hbase_table,
+ self.__params.update_hbase_cf)
Logger.info("Done creating HBase Tables for indexing")
self.set_hbase_configured()
http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index d8b8b10..330d3c0 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -282,6 +282,32 @@ def check_kafka_topics(params, topics):
err_msg = "Missing Kafka topic; topic={0}".format(topic)
execute(cmd, user=params.kafka_user, err_msg=err_msg)
+
+def create_hbase_table(params, table, cf):
+ """
+ Creates an HBase table, if the table does not currently exist
+ :param params:
+ :param table: The name of the HBase table.
+ :param cf: The column family
+ :param user: The user to execute the command as
+ """
+ if params.security_enabled:
+ kinit(params.kinit_path_local,
+ params.hbase_keytab_path,
+ params.hbase_principal_name,
+ execute_user=params.hbase_user)
+ cmd = """if [[ $(echo \"exists '{0}'\" | hbase shell | grep 'not exist') ]]; \
+ then echo \"create '{0}','{1}'\" | hbase shell -n; fi"""
+ add_update_cmd = cmd.format(table, cf)
+ Execute(add_update_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ user=params.hbase_user
+ )
+
+
def check_hbase_table(params, table):
"""
Validates that an HBase table exists. An exception is raised if the table
http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
index 41cab06..b026a30 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
@@ -18,13 +18,12 @@ limitations under the License.
import os
import time
-from datetime import datetime
from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
+from resource_management.core.resources.system import Execute
-import metron_service
import metron_security
+import metron_service
# Wrap major operations and functionality in this class
@@ -79,22 +78,11 @@ class ProfilerCommands:
metron_service.set_configured(self.__params.metron_user, self.__params.profiler_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for profiler")
def create_hbase_tables(self):
- Logger.info("Creating HBase table '{0}' for profiler".format(self.__params.profiler_hbase_table))
- if self.__params.security_enabled:
- metron_security.kinit(self.__params.kinit_path_local,
- self.__params.hbase_keytab_path,
- self.__params.hbase_principal_name,
- execute_user=self.__params.hbase_user)
- cmd = "echo \"create '{0}','{1}'\" | hbase shell -n"
- add_table_cmd = cmd.format(self.__params.profiler_hbase_table, self.__params.profiler_hbase_cf)
- Execute(add_table_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- user=self.__params.hbase_user
- )
-
+ Logger.info("Creating HBase table '{0}' for profiler".format(
+ self.__params.profiler_hbase_table))
+ metron_service.create_hbase_table(self.__params,
+ self.__params.profiler_hbase_table,
+ self.__params.profiler_hbase_cf)
self.set_hbase_configured()
Logger.info("Done creating HBase Tables for profiler")