You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:45 UTC
[29/50] [abbrv] metron git commit: METRON-1609 Elasticsearch settings
in Ambari should not be required if Solr is the indexer (nickwallen) closes
apache/metron#1056
METRON-1609 Elasticsearch settings in Ambari should not be required if Solr is the indexer (nickwallen) closes apache/metron#1056
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ed4dfb93
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ed4dfb93
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ed4dfb93
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: ed4dfb93641c9c9a84b61b5daddbebd962b3b20c
Parents: 2bf6650
Author: nickwallen <ni...@nickallen.org>
Authored: Thu Jun 21 11:28:30 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Thu Jun 21 11:28:30 2018 -0400
----------------------------------------------------------------------
.../METRON/CURRENT/configuration/metron-env.xml | 32 ++++--
.../package/scripts/indexing_commands.py | 4 +-
.../CURRENT/package/scripts/indexing_master.py | 10 +-
.../CURRENT/package/scripts/metron_service.py | 101 ++++++++++++++-----
4 files changed, 113 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 0b64f3a..cdef7cf 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -81,30 +81,54 @@
<value>metron</value>
<description>Name of Elasticsearch Cluster</description>
<display-name>Elasticsearch Cluster Name</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
</property>
- <property require-input="true">
+ <property>
<name>es_hosts</name>
<value></value>
<description>Comma delimited list of Elasticsearch Master Hosts: eshost1,eshost2</description>
<display-name>Elasticsearch Hosts</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
</property>
<property>
<name>es_binary_port</name>
<value>9300</value>
<description>Elasticsearch binary port. (9300)</description>
<display-name>Elasticsearch Binary Port</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
</property>
<property>
<name>es_http_port</name>
<value>9200</value>
<description>Elasticsearch HTTP port. (9200)</description>
<display-name>Elasticsearch HTTP port</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>es_date_format</name>
+ <description>Elasticsearch Date Format</description>
+ <value>yyyy.MM.dd.HH</value>
+ <display-name>Elasticsearch Date Format</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
</property>
<property>
<name>solr_zookeeper_url</name>
<value>{{zookeeper_quorum}}</value>
<description>Comma delimited list of Zookeeper Urls: zkhost1:2181,zkhost1:2181</description>
<display-name>Solr Zookeeper Urls</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
</property>
<property require-input = "true">
<name>storm_rest_addr</name>
@@ -137,10 +161,4 @@
<empty-value-valid>true</empty-value-valid>
</value-attributes>
</property>
- <property>
- <name>es_date_format</name>
- <description>Elasticsearch Date Format</description>
- <value>yyyy.MM.dd.HH</value>
- <display-name>Elasticsearch Date Format</display-name>
- </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 69e980b..4802add 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -30,7 +30,6 @@ from resource_management.libraries.functions.format import format
import metron_service
import metron_security
-
# Wrap major operations and functionality in this class
class IndexingCommands:
__params = None
@@ -394,7 +393,6 @@ class IndexingCommands:
is_random_access_running = topologies[self.__random_access_indexing_topology] in ['ACTIVE', 'REBALANCING']
return is_random_access_running
-
def is_topology_active(self, env):
return self.is_batch_topology_active(env) and self.is_random_access_topology_active(env)
@@ -403,6 +401,8 @@ class IndexingCommands:
Performs a service check for Indexing.
:param env: Environment
"""
+ metron_service.check_indexer_parameters()
+
Logger.info('Checking Kafka topics for Indexing')
metron_service.check_kafka_topics(self.__params, self.__get_topics())
http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 1629465..9f9ab87 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -48,6 +48,7 @@ class Indexing(Script):
env.set_params(params)
Logger.info("Running indexing configure")
+ metron_service.check_indexer_parameters()
File(format("{metron_config_path}/elasticsearch.properties"),
content=Template("elasticsearch.properties.j2"),
owner=params.metron_user,
@@ -137,6 +138,7 @@ class Indexing(Script):
def restart(self, env):
from params import params
env.set_params(params)
+
self.configure(env)
commands = IndexingCommands(params)
commands.restart_indexing_topology(env)
@@ -145,6 +147,7 @@ class Indexing(Script):
from params import params
env.set_params(params)
Logger.info("Installing Elasticsearch index templates")
+ metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
for template_name, template_path in commands.get_templates().iteritems():
@@ -160,9 +163,11 @@ class Indexing(Script):
from params import params
env.set_params(params)
Logger.info("Deleting Elasticsearch index templates")
+ metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
for template_name in commands.get_templates():
+
# delete the index template
cmd = "curl -s -XDELETE \"http://{0}/_template/{1}\""
Execute(
@@ -173,9 +178,9 @@ class Indexing(Script):
def kibana_dashboard_install(self, env):
from params import params
env.set_params(params)
+ metron_service.check_indexer_parameters()
Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url))
-
kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template')
if not os.path.isfile(kibanaTemplate):
raise IOError(
@@ -200,8 +205,9 @@ class Indexing(Script):
def zeppelin_notebook_import(self, env):
from params import params
env.set_params(params)
- commands = IndexingCommands(params)
+ metron_service.check_indexer_parameters()
+ commands = IndexingCommands(params)
Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'))
# Check if authentication is configured on Zeppelin server, and fetch details if enabled.
http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index 23482e3..4a55f8a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -27,6 +27,7 @@ from resource_management.core.resources.system import Execute
from resource_management.core.source import InlineTemplate
from resource_management.libraries.functions import format as ambari_format
from resource_management.libraries.functions.get_user_call_output import get_user_call_output
+from resource_management.libraries.script import Script
from metron_security import kinit
@@ -51,10 +52,23 @@ def set_configured(user, flag_file, log_msg):
def set_zk_configured(params):
set_configured(params.metron_user, params.zk_configured_flag_file, "Setting Zookeeper configured to true")
-def build_global_config_patch(params, patch_file):
- # see RFC 6902 at https://tools.ietf.org/html/rfc6902
- patch_template = """
- [
+def solr_global_config_patches():
+ """
+ Builds the global configuration patches required for Solr.
+ """
+ return """
+ {
+ "op": "add",
+ "path": "/solr.zookeeper",
+ "value": "{{solr_zookeeper_url}}"
+ }
+ """
+
+def elasticsearch_global_config_patches():
+ """
+ Builds the global configuration patches required for Elasticsearch.
+ """
+ return """
{
"op": "add",
"path": "/es.clustername",
@@ -69,11 +83,31 @@ def build_global_config_patch(params, patch_file):
"op": "add",
"path": "/es.date.format",
"value": "{{es_date_format}}"
+ }
+ """
+
+def build_global_config_patch(params, patch_file):
+ """
+ Build the file used to patch the global configuration.
+ See RFC 6902 at https://tools.ietf.org/html/rfc6902
+
+ :param params:
+ :param patch_file: The path where the patch file will be created.
+ """
+ if params.ra_indexing_writer == 'Solr':
+ indexing_patches = solr_global_config_patches()
+ else:
+ indexing_patches = elasticsearch_global_config_patches()
+ other_patches = """
+ {
+ "op": "add",
+ "path": "/profiler.client.period.duration",
+ "value": "{{profiler_period_duration}}"
},
{
"op": "add",
- "path": "/solr.zookeeper",
- "value": "{{solr_zookeeper_url}}"
+ "path": "/profiler.client.period.duration.units",
+ "value": "{{profiler_period_units}}"
},
{
"op": "add",
@@ -92,16 +126,6 @@ def build_global_config_patch(params, patch_file):
},
{
"op": "add",
- "path": "/profiler.client.period.duration",
- "value": "{{profiler_period_duration}}"
- },
- {
- "op": "add",
- "path": "/profiler.client.period.duration.units",
- "value": "{{profiler_period_units}}"
- },
- {
- "op": "add",
"path": "/user.settings.hbase.table",
"value": "{{user_settings_hbase_table}}"
},
@@ -115,8 +139,14 @@ def build_global_config_patch(params, patch_file):
"path": "/bootstrap.servers",
"value": "{{kafka_brokers}}"
}
- ]
"""
+ patch_template = ambari_format(
+ """
+ [
+ {indexing_patches},
+ {other_patches}
+ ]
+ """)
File(patch_file,
content=InlineTemplate(patch_template),
owner=params.metron_user,
@@ -132,6 +162,7 @@ def patch_global_config(params):
"{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PATCH --config_type GLOBAL --patch_file " + patch_file),
path=ambari_format("{java_home}/bin")
)
+ Logger.info("Done patching global config")
def pull_config(params):
Logger.info('Pulling all Metron configs down from ZooKeeper to local file system')
@@ -141,17 +172,12 @@ def pull_config(params):
path=ambari_format("{java_home}/bin")
)
-# pushes json patches to zookeeper based on Ambari parameters that are configurable by the user
def refresh_configs(params):
if not is_zk_configured(params):
Logger.warning("The expected flag file '" + params.zk_configured_flag_file + "'indicating that Zookeeper has been configured does not exist. Skipping patching. An administrator should look into this.")
return
-
- Logger.info("Patch global config in Zookeeper")
+ check_indexer_parameters()
patch_global_config(params)
- Logger.info("Done patching global config")
-
- Logger.info("Pull zookeeper config locally")
pull_config(params)
def get_running_topologies(params):
@@ -505,3 +531,32 @@ def check_http(host, port, user):
Execute(cmd, tries=3, try_sleep=5, logoutput=False, user=user)
except:
raise ComponentIsNotRunning()
+
+def check_indexer_parameters():
+ """
+ Ensure that all required parameters have been defined for the chosen
+ Indexer; either Solr or Elasticsearch.
+ """
+ missing = []
+ config = Script.get_config()
+ indexer = config['configurations']['metron-indexing-env']['ra_indexing_writer']
+ Logger.info('Checking parameters for indexer = ' + indexer)
+
+ if indexer == 'Solr':
+ # check for all required solr parameters
+ if not config['configurations']['metron-env']['solr_zookeeper_url']:
+ missing.append("metron-env/solr_zookeeper_url")
+
+ else:
+ # check for all required elasticsearch parameters
+ if not config['configurations']['metron-env']['es_cluster_name']:
+ missing.append("metron-env/es_cluster_name")
+ if not config['configurations']['metron-env']['es_hosts']:
+ missing.append("metron-env/es_hosts")
+ if not config['configurations']['metron-env']['es_binary_port']:
+ missing.append("metron-env/es_binary_port")
+ if not config['configurations']['metron-env']['es_date_format']:
+ missing.append("metron-env/es_date_format")
+
+ if len(missing) > 0:
+ raise Fail("Missing required indexing parameters(s): indexer={0}, missing={1}".format(indexer, missing))