You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2017/11/16 13:42:52 UTC
metron git commit: METRON-1311 Service Check Should Check
Elasticsearch Index Templates (nickwallen) closes apache/metron#839
Repository: metron
Updated Branches:
refs/heads/master fd896fbeb -> 2d5209ebf
METRON-1311 Service Check Should Check Elasticsearch Index Templates (nickwallen) closes apache/metron#839
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2d5209eb
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2d5209eb
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2d5209eb
Branch: refs/heads/master
Commit: 2d5209ebf7b7876a7da8a3908cb7808f0ad22615
Parents: fd896fb
Author: nickwallen <ni...@nickallen.org>
Authored: Thu Nov 16 08:42:04 2017 -0500
Committer: nickallen <ni...@apache.org>
Committed: Thu Nov 16 08:42:04 2017 -0500
----------------------------------------------------------------------
.../CURRENT/package/files/meta_index.template | 47 --------------
.../package/files/metaalert_index.template | 47 ++++++++++++++
.../package/scripts/indexing_commands.py | 29 +++++++++
.../CURRENT/package/scripts/indexing_master.py | 68 +++++---------------
.../package/scripts/params/params_linux.py | 2 +-
5 files changed, 94 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/2d5209eb/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template
deleted file mode 100644
index 964a480..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "template": "metaalert_index*",
- "mappings": {
- "metaalert_doc": {
- "_timestamp": {
- "enabled": true
- },
- "dynamic_templates": [
- {
- "alert_template": {
- "path_match": "alert.*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- }
- ],
- "properties": {
- "guid": {
- "type": "string",
- "index": "not_analyzed"
- },
- "score": {
- "type": "string",
- "index": "not_analyzed"
- },
- "status": {
- "type": "string",
- "index": "not_analyzed"
- },
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "alert": {
- "type": "nested"
- },
- "source:type": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/2d5209eb/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
new file mode 100644
index 0000000..964a480
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
@@ -0,0 +1,47 @@
+{
+ "template": "metaalert_index*",
+ "mappings": {
+ "metaalert_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "alert_template": {
+ "path_match": "alert.*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "guid": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "score": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "status": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "alert": {
+ "type": "nested"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/2d5209eb/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index e6cfabb..5a2b0f4 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -58,6 +58,21 @@ class IndexingCommands:
# Indexed topic names matches the group
return [self.__indexing_topic]
+ def get_templates(self):
+ """
+ Defines the Elasticsearch index templates.
+ :return: Dict where key is the name of an index template and the
+ value is a path to file containing the index template definition.
+ """
+ from params import params
+ return {
+ "bro_index": params.bro_index_path,
+ "yaf_index": params.yaf_index_path,
+ "snort_index": params.snort_index_path,
+ "error_index": params.error_index_path,
+ "metaalert_index": params.meta_index_path
+ }
+
def is_configured(self):
return self.__configured
@@ -159,6 +174,17 @@ class IndexingCommands:
)
Logger.info('Done creating HDFS indexing directory')
+ def check_elasticsearch_templates(self):
+ for template_name in self.get_templates():
+
+ # check for the index template
+ cmd = "curl -s -XGET \"http://{0}/_template/{1}\" | grep -o {1}"
+ err_msg="Missing Elasticsearch index template: name={0}"
+ metron_service.execute(
+ cmd=cmd.format(self.__params.es_http_url, template_name),
+ user=self.__params.metron_user,
+ err_msg=err_msg.format(template_name))
+
def start_indexing_topology(self, env):
Logger.info('Starting ' + self.__indexing_topology)
@@ -241,6 +267,9 @@ class IndexingCommands:
metron_service.check_hbase_table(self.__params, self.__params.update_hbase_table)
metron_service.check_hbase_column_family(self.__params, self.__params.update_hbase_table, self.__params.update_hbase_cf)
+ Logger.info('Checking Elasticsearch templates for Indexing')
+ self.check_elasticsearch_templates()
+
if self.__params.security_enabled:
Logger.info('Checking Kafka ACLs for Indexing')
http://git-wip-us.apache.org/repos/asf/metron/blob/2d5209eb/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 92077ac..8992950 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -118,65 +118,31 @@ class Indexing(Script):
def elasticsearch_template_install(self, env):
from params import params
env.set_params(params)
+ Logger.info("Installing Elasticsearch index templates")
- File(params.bro_index_path,
- mode=0755,
- content=StaticFile('bro_index.template')
- )
-
- File(params.snort_index_path,
- mode=0755,
- content=StaticFile('snort_index.template')
- )
-
- File(params.yaf_index_path,
- mode=0755,
- content=StaticFile('yaf_index.template')
- )
-
- File(params.error_index_path,
- mode=0755,
- content=StaticFile('error_index.template')
- )
-
- File(params.meta_index_path,
- mode=0755,
- content=StaticFile('meta_index.template')
- )
-
- bro_cmd = ambari_format('curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}')
- Execute(bro_cmd, logoutput=True)
-
- snort_cmd = ambari_format('curl -s -XPOST http://{es_http_url}/_template/snort_index -d @{snort_index_path}')
- Execute(snort_cmd, logoutput=True)
-
- yaf_cmd = ambari_format('curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}')
- Execute(yaf_cmd, logoutput=True)
-
- error_cmd = ambari_format('curl -s -XPOST http://{es_http_url}/_template/error_index -d @{error_index_path}')
- Execute(error_cmd, logoutput=True)
+ commands = IndexingCommands(params)
+ for template_name, template_path in commands.get_templates().iteritems():
- meta_cmd = ambari_format('curl -s -XPOST http://{es_http_url}/_template/metaalert_index -d @{meta_index_path}')
- Execute(meta_cmd, logoutput=True)
+ # install the index template
+ File(template_path, mode=0755, content=StaticFile("{0}.template".format(template_name)))
+ cmd = "curl -s -XPOST http://{0}/_template/{1} -d @{2}"
+ Execute(
+ cmd.format(params.es_http_url, template_name, template_path),
+ logoutput=True)
def elasticsearch_template_delete(self, env):
from params import params
env.set_params(params)
+ Logger.info("Deleting Elasticsearch index templates")
- bro_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/_template/bro_index"')
- Execute(bro_cmd, logoutput=True)
-
- snort_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/_template/snort_index"')
- Execute(snort_cmd, logoutput=True)
-
- yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/_template/yaf_index"')
- Execute(yaf_cmd, logoutput=True)
-
- error_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/_template/error_index"')
- Execute(error_cmd, logoutput=True)
+ commands = IndexingCommands(params)
+ for template_name in commands.get_templates():
- meta_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/_template/metaalert_index"')
- Execute(meta_cmd, logoutput=True)
+ # delete the index template
+ cmd = "curl -s -XDELETE \"http://{0}/_template/{1}\""
+ Execute(
+ cmd.format(params.es_http_url, template_name),
+ logoutput=True)
def zeppelin_notebook_import(self, env):
from params import params
http://git-wip-us.apache.org/repos/asf/metron/blob/2d5209eb/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 077a9c1..32d8889 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -195,7 +195,7 @@ bro_index_path = tmp_dir + "/bro_index.template"
snort_index_path = tmp_dir + "/snort_index.template"
yaf_index_path = tmp_dir + "/yaf_index.template"
error_index_path = tmp_dir + "/error_index.template"
-meta_index_path = tmp_dir + "/meta_index.template"
+meta_index_path = tmp_dir + "/metaalert_index.template"
# Zeppelin Notebooks
metron_config_zeppelin_path = format("{metron_config_path}/zeppelin")