You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/04/20 04:46:52 UTC
incubator-metron git commit: METRON-634 Mpack bug fixes and
improvements (not related to singlenode install) (mattf-horton) closes
apache/incubator-metron#532
Repository: incubator-metron
Updated Branches:
refs/heads/master 29d7cb378 -> df44f077a
METRON-634 Mpack bug fixes and improvements (not related to singlenode install) (mattf-horton) closes apache/incubator-metron#532
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/df44f077
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/df44f077
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/df44f077
Branch: refs/heads/master
Commit: df44f077a97d0495fea511f5d9f900e4d3a25147
Parents: 29d7cb3
Author: mattf-horton <mf...@hortonworks.com>
Authored: Wed Apr 19 21:43:05 2017 -0700
Committer: mattf <ma...@apache.org>
Committed: Wed Apr 19 21:43:05 2017 -0700
----------------------------------------------------------------------
.../2.3.3/configuration/elastic-site.xml | 33 ++++++++-------
.../2.3.3/configuration/elastic-sysconfig.xml | 8 +++-
.../ELASTICSEARCH/2.3.3/metainfo.xml | 8 +++-
.../2.3.3/package/scripts/elastic.py | 6 +--
.../2.3.3/package/scripts/elastic_master.py | 8 ++--
.../2.3.3/package/scripts/elastic_slave.py | 8 ++--
.../2.3.3/package/scripts/params.py | 4 +-
.../2.3.3/quicklinks/quicklinks.json | 43 ++++++++++++++++++++
.../METRON/CURRENT/configuration/metron-env.xml | 2 +-
9 files changed, 87 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
index cbcebdd..fd29e63 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
@@ -24,7 +24,7 @@
<property>
<name>cluster_name</name>
<value>metron</value>
- <description>Cluster name identifies your cluster</description>
+ <description>Elasticsearch Cluster Name identifies your Elasticsearch subsystem</description>
</property>
<property>
<name>masters_also_are_datanodes</name>
@@ -37,24 +37,24 @@
<property>
<name>zen_discovery_ping_unicast_hosts</name>
<!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.-->
+ <!--Also need to document whether should list masters only, or all ES nodes. I think this one is all nodes, but previous inline comment said Masters.-->
<value></value>
- <description>Unicast discovery list of hosts to act as gossip routers, in comma separated format.</description>
+ <description>Unicast discovery list of hosts to act as gossip routers, in comma separated format: eshost1,eshost2</description>
</property>
<property>
<name>index_number_of_shards</name>
<value>4</value>
- <description>Set the number of shards (splits) of an index</description>
+ <description>Set the number of shards (splits) of an index. Changes are not effective after index creation. Usually set to 1 for single-node install.</description>
</property>
<property>
<name>index_number_of_replicas</name>
<value>2</value>
- <description>Set the number of replicas (additional copies) of an index</description>
+ <description>Set the number of replicas (copies in addition to the first) of an index. Usually set to 0 for single-node install.</description>
</property>
- <!-- Logging Configurations -->
<property>
<name>path_data</name>
<value>"/opt/lmm/es_data"</value>
- <description>Path to directory where to store index data allocated for this node. e.g. "/mnt/first", "/mnt/second"</description>
+ <description>Comma-separated list of directories where to store index data allocated for each node: "/mnt/first","/mnt/second". Number of paths should relate to number of shards, and preferably should be on separate physical volumes.</description>
</property>
<property>
<name>http_cors_enabled</name>
@@ -64,22 +64,21 @@
<type>string</type>
</value-attributes>
</property>
- <!-- Discovery -->
- <property>
- <name>transport_tcp_port</name>
- <value>9300-9400</value>
- <description>Set a custom port for the node to node communication</description>
- </property>
<property>
<name>http_port</name>
<value>9200-9300</value>
<description>Set a custom port to listen for HTTP traffic</description>
</property>
- <!-- Discovery -->
+ <property>
+ <name>transport_tcp_port</name>
+ <value>9300-9400</value>
+ <description>Set a custom port for the node to node communication</description>
+ </property>
+ <!-- Multi-node Discovery -->
<property>
<name>discovery_zen_ping_multicast_enabled</name>
<value>false</value>
- <description>master eligible nodes</description>
+ <description>Whether to use multicast</description>
</property>
<property>
<name>discovery_zen_ping_timeout</name>
@@ -190,7 +189,7 @@
</property>
<property>
<name>network_host</name>
- <value>_lo_,_eth0_</value>
- <description>Network interface(s) will bind to. </description>
+ <value>"_lo:ipv4_","_eth0:ipv4_"</value>
+ <description>Network interface(s) ES will bind to within each node. Confirm names via ifconfig. Should have entries for primary external and loopback interfaces, with :ipv4 annotation and quote marks around each entry: "_lo:ipv4_","_eth0:ipv4_"</description>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
index ae26498..44ecf05 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
@@ -58,7 +58,7 @@
<!--/etc/sysconfig/elasticsearch-->
<property>
<name>content</name>
- <description>This is the jinja template for elastic-env.sh file</description>
+ <description>This is the jinja template for elastic sysconfig file (/etc/sysconfig/elasticsearch)</description>
<value>
# Directory where the Elasticsearch binary distribution resides
ES_HOME={{elastic_home}}
@@ -89,6 +89,12 @@ CONF_DIR={{conf_dir}}
# This setting only works for the init script, but has to be configured separately for systemd startup
ES_USER={{elastic_user}}
+# Elasticsearch pid directory
+PID_DIR={{pid_dir}}
+
+# JAVA_HOME must be provided here for OS that use systemd service launch
+JAVA_HOME={{java64_home}}
+
# Additional Java OPTS
ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled \
-XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps \
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
index 53909a2..4373e38 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
@@ -68,6 +68,12 @@
<config-type>elastic-sysconfig</config-type>
</configuration-dependencies>
<restartRequiredAfterChange>true</restartRequiredAfterChange>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
</service>
</services>
-</metainfo>
\ No newline at end of file
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
index 2e638de..c44d5ef 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
@@ -33,7 +33,7 @@ def elastic():
data_path[:] = [x.replace('"', '') for x in data_path]
directories = [params.log_dir, params.pid_dir, params.conf_dir]
- directories = directories + data_path
+ directories = directories + data_path + ["{0}/scripts".format(params.conf_dir)]
Directory(directories,
create_parents=True,
@@ -57,11 +57,11 @@ def elastic():
"elasticsearch.master.yaml.j2",
configurations=configurations),
owner=params.elastic_user,
- group=params.elastic_user
+ group=params.elastic_group
)
print "Master sysconfig: /etc/sysconfig/elasticsearch"
- File(format("/etc/sysconfig/elasticsearch"),
+ File("/etc/sysconfig/elasticsearch",
owner="root",
group="root",
content=InlineTemplate(params.sysconfig_template)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
index 706610e..3cd63c3 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
@@ -40,7 +40,7 @@ class Elasticsearch(Script):
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
- stop_cmd = format("service elasticsearch stop")
+ stop_cmd = "service elasticsearch stop"
print 'Stop the Master'
Execute(stop_cmd)
@@ -49,14 +49,14 @@ class Elasticsearch(Script):
env.set_params(params)
self.configure(env)
- start_cmd = format("service elasticsearch start")
+ start_cmd = "service elasticsearch start"
print 'Start the Master'
Execute(start_cmd)
def status(self, env):
import params
env.set_params(params)
- status_cmd = format("service elasticsearch status")
+ status_cmd = "service elasticsearch status"
print 'Status of the Master'
Execute(status_cmd)
@@ -64,7 +64,7 @@ class Elasticsearch(Script):
import params
env.set_params(params)
self.configure(env)
- restart_cmd = format("service elasticsearch restart")
+ restart_cmd = "service elasticsearch restart"
print 'Restarting the Master'
Execute(restart_cmd)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
index 7f7f21a..633ddd9 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
@@ -39,7 +39,7 @@ class Elasticsearch(Script):
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
- stop_cmd = format("service elasticsearch stop")
+ stop_cmd = "service elasticsearch stop"
print 'Stop the Slave'
Execute(stop_cmd)
@@ -47,14 +47,14 @@ class Elasticsearch(Script):
import params
env.set_params(params)
self.configure(env)
- start_cmd = format("service elasticsearch start")
+ start_cmd = "service elasticsearch start"
print 'Start the Slave'
Execute(start_cmd)
def status(self, env):
import params
env.set_params(params)
- status_cmd = format("service elasticsearch status")
+ status_cmd = "service elasticsearch status"
print 'Status of the Slave'
Execute(status_cmd)
@@ -62,7 +62,7 @@ class Elasticsearch(Script):
import params
env.set_params(params)
self.configure(env)
- restart_cmd = format("service elasticsearch restart")
+ restart_cmd = "service elasticsearch restart"
print 'Restarting the Slave'
Execute(restart_cmd)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
index aea81fb..00929a7 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
@@ -35,8 +35,8 @@ max_map_count = config['configurations']['elastic-sysconfig']['max_map_count']
elastic_user = config['configurations']['elastic-env']['elastic_user']
elastic_group = config['configurations']['elastic-env']['elastic_group']
log_dir = config['configurations']['elastic-env']['elastic_log_dir']
-pid_dir = '/var/run/elasticsearch'
-pid_file = '/var/run/elasticsearch/elasticsearch.pid'
+pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
+
hostname = config['hostname']
java64_home = config['hostLevelParams']['java_home']
elastic_env_sh_template = config['configurations']['elastic-env']['content']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json
new file mode 100644
index 0000000..909828b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json
@@ -0,0 +1,43 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "es_health_link",
+ "label": "Elasticsearch Health",
+ "requires_user_name": "false",
+ "component_name": "ES_MASTER",
+ "url":"%@://%@:%@/_cat/health?v",
+ "port":{
+ "http_property": "http_port",
+ "http_default_port": "9200",
+ "https_property": "http_port",
+ "https_default_port": "9200",
+ "regex": "^(\\d+)",
+ "site": "elastic-site"
+ }
+ },
+ {
+ "name": "es_indices_link",
+ "label": "Elasticsearch Indexes",
+ "requires_user_name": "false",
+ "component_name": "ES_MASTER",
+ "url":"%@://%@:%@/_cat/indices?v",
+ "port":{
+ "http_property": "http_port",
+ "http_default_port": "9200",
+ "https_property": "http_port",
+ "https_default_port": "9200",
+ "regex": "^(\\d+)",
+ "site": "elastic-site"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 90e0757..218e8e5 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -97,7 +97,7 @@
<property require-input="true">
<name>es_hosts</name>
<value></value>
- <description>Comma delimited list of Elasticsearch Hosts. (eshost1,eshost2)</description>
+ <description>Comma delimited list of Elasticsearch Master Hosts: eshost1,eshost2</description>
<display-name>Elasticsearch Hosts</display-name>
</property>
<property>