You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2017/10/30 16:23:52 UTC
metron git commit: METRON-1283 Install Elasticsearch template as a
part of the mpack startup scripts (anandsubbu via nickwallen) closes
apache/metron#817
Repository: metron
Updated Branches:
refs/heads/master 191d5d84a -> 2488842bd
METRON-1283 Install Elasticsearch template as a part of the mpack startup scripts (anandsubbu via nickwallen) closes apache/metron#817
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2488842b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2488842b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2488842b
Branch: refs/heads/master
Commit: 2488842bdfc8bf394558be93bc9bb6017759fa57
Parents: 191d5d8
Author: anandsubbu <as...@hortonworks.com>
Authored: Mon Oct 30 12:23:27 2017 -0400
Committer: nickallen <ni...@apache.org>
Committed: Mon Oct 30 12:23:27 2017 -0400
----------------------------------------------------------------------
.../CURRENT/package/scripts/indexing_commands.py | 7 +++++++
.../CURRENT/package/scripts/indexing_master.py | 7 +++++++
.../CURRENT/package/scripts/params/params_linux.py | 1 +
.../package/scripts/params/status_params.py | 3 +++
.../roles/load_web_templates/tasks/main.yml | 8 --------
metron-platform/metron-elasticsearch/README.md | 17 ++++++++++++++++-
6 files changed, 34 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 17374eb..e6cfabb 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -48,6 +48,7 @@ class IndexingCommands:
self.__acl_configured = os.path.isfile(self.__params.indexing_acl_configured_flag_file)
self.__hbase_configured = os.path.isfile(self.__params.indexing_hbase_configured_flag_file)
self.__hbase_acl_configured = os.path.isfile(self.__params.indexing_hbase_acl_configured_flag_file)
+ self.__elasticsearch_template_installed = os.path.isfile(self.__params.elasticsearch_template_installed_flag_file)
self.__hdfs_perm_configured = os.path.isfile(self.__params.indexing_hdfs_perm_configured_flag_file)
def __get_topics(self):
@@ -72,6 +73,9 @@ class IndexingCommands:
def is_hbase_acl_configured(self):
return self.__hbase_acl_configured
+ def is_elasticsearch_template_installed(self):
+ return self.__elasticsearch_template_installed
+
def set_configured(self):
metron_service.set_configured(self.__params.metron_user, self.__params.indexing_configured_flag_file, "Setting Indexing configured to True")
@@ -87,6 +91,9 @@ class IndexingCommands:
def set_hdfs_perm_configured(self):
metron_service.set_configured(self.__params.metron_user, self.__params.indexing_hdfs_perm_configured_flag_file, "Setting HDFS perm configured to True")
+ def set_elasticsearch_template_installed(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.elasticsearch_template_installed_flag_file, "Setting Elasticsearch template installed to True")
+
def create_hbase_tables(self):
Logger.info("Creating HBase Tables for indexing")
if self.__params.security_enabled:
http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 8f156d6..e88b949 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -80,6 +80,13 @@ class Indexing(Script):
from params import params
env.set_params(params)
self.configure(env)
+ # Install elasticsearch templates
+ try:
+ if not commands.is_elasticsearch_template_installed():
+ self.elasticsearch_template_install(env)
+ commands.set_elasticsearch_template_installed()
+ except:
+ Logger.warning("WARNING: Elasticsearch templates could not be installed. The Elasticsearch service is probably down.")
commands = IndexingCommands(params)
commands.start_indexing_topology(env)
http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index a181935..fa4c0ab 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -80,6 +80,7 @@ indexing_acl_configured_flag_file = status_params.indexing_acl_configured_flag_f
indexing_hbase_configured_flag_file = status_params.indexing_hbase_configured_flag_file
indexing_hbase_acl_configured_flag_file = status_params.indexing_hbase_acl_configured_flag_file
indexing_hdfs_perm_configured_flag_file = status_params.indexing_hdfs_perm_configured_flag_file
+elasticsearch_template_installed_flag_file = status_params.elasticsearch_template_installed_flag_file
global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
# Elasticsearch hosts and port management
http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index d4f2799..a9a37d3 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -74,6 +74,9 @@ indexing_hdfs_perm_configured_flag_file = metron_zookeeper_config_path + '/../me
indexing_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_hbase_configured'
indexing_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_hbase_acl_configured'
+# Elasticsearch
+elasticsearch_template_installed_flag_file = metron_zookeeper_config_path + '/../metron_elasticsearch_template_installed_flag_file'
+
# REST
metron_rest_port = config['configurations']['metron-rest-env']['metron_rest_port']
http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-deployment/roles/load_web_templates/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/load_web_templates/tasks/main.yml b/metron-deployment/roles/load_web_templates/tasks/main.yml
index a5dbbba..3a91960 100644
--- a/metron-deployment/roles/load_web_templates/tasks/main.yml
+++ b/metron-deployment/roles/load_web_templates/tasks/main.yml
@@ -15,14 +15,6 @@
# limitations under the License.
#
---
-- name: Load ES Templates
- command: >
- curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install ES Template from REST", "command": "ELASTICSEARCH_TEMPLATE_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "{{ metron_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests
- args:
- warn: off
- register: result
- failed_when: "result.rc != 0 or '202' not in result.stdout"
-
- name: Load Kibana Dashboard
command: >
curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "{{ kibana_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests
http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-platform/metron-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md
index 1e39691..9113672 100644
--- a/metron-platform/metron-elasticsearch/README.md
+++ b/metron-platform/metron-elasticsearch/README.md
@@ -2,7 +2,7 @@
## Introduction
-Elasticsearch can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing.README.md).
+Elasticsearch can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md).
## Properties
@@ -81,3 +81,18 @@ curl -XPUT "http://${ELASTICSEARCH}:9200/${SENSOR}_index*/_mapping/${SENSOR}_doc
'
rm ${SENSOR}.template
```
+
+## Installing Elasticsearch Templates
+
+The stock set of Elasticsearch templates for bro, snort, yaf, error index and meta index are installed automatically during the first time install and startup of Metron Indexing service.
+
+It is possible that Elasticsearch service is not available when the Metron Indexing Service startup, in that case the Elasticsearch template will not be installed.
+
+For such a scenario, an Admin can have the template installed in two ways:
+
+_Method 1_ - Manually from the Ambari UI by following the flow:
+Ambari UI -> Services -> Metron -> Service Actions -> Elasticsearch Template Install
+
+_Method 2_ - Stop the Metron Indexing service, and start it again from Ambari UI. Note that the Metron Indexing service tracks if it has successfully installed the Elasticsearch templates, and will attempt to do so each time it is Started until successful.
+
+> Note: If you have made any customization to your index templates, then installing Elasticsearch templates afresh will lead to overwriting your existing changes. Please exercise caution.