You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2018/04/27 19:30:03 UTC
[23/50] [abbrv] metron git commit: METRON-1462: Separate ES and
Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
new file mode 100755
index 0000000..24f2306
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+
+def yamlify_variables(var) :
+ if isinstance(var, type(True)):
+ return str(var).lower()
+ else:
+ return var
+
+# server configurations
+config = Script.get_config()
+
+masters_also_are_datanodes = config['configurations']['elastic-site']['masters_also_are_datanodes']
+elastic_home = config['configurations']['elastic-sysconfig']['elastic_home']
+data_dir = config['configurations']['elastic-sysconfig']['data_dir']
+work_dir = config['configurations']['elastic-sysconfig']['work_dir']
+conf_dir = config['configurations']['elastic-sysconfig']['conf_dir']
+heap_size = config['configurations']['elastic-sysconfig']['heap_size']
+max_open_files = config['configurations']['elastic-sysconfig']['max_open_files']
+max_map_count = config['configurations']['elastic-sysconfig']['max_map_count']
+
+elastic_user = config['configurations']['elastic-env']['elastic_user']
+elastic_group = config['configurations']['elastic-env']['elastic_group']
+log_dir = config['configurations']['elastic-env']['elastic_log_dir']
+pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
+
+hostname = config['hostname']
+java64_home = config['hostLevelParams']['java_home']
+elastic_env_sh_template = config['configurations']['elastic-env']['content']
+sysconfig_template = config['configurations']['elastic-sysconfig']['content']
+
+cluster_name = config['configurations']['elastic-site']['cluster_name']
+zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts']
+
+path_data = config['configurations']['elastic-site']['path_data']
+http_cors_enabled = config['configurations']['elastic-site']['http_cors_enabled']
+http_port = config['configurations']['elastic-site']['http_port']
+transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port']
+
+recover_after_time = config['configurations']['elastic-site']['recover_after_time']
+gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes']
+expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes']
+index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count']
+index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size']
+index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval']
+indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type']
+index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards']
+index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas']
+indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size']
+bootstrap_memory_lock = yamlify_variables(config['configurations']['elastic-site']['bootstrap_memory_lock'])
+threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size']
+cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries']
+cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low']
+cluster_routing_allocation_disk_threshold_enabled = yamlify_variables(config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled'])
+cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high']
+indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size']
+indices_cluster_send_refresh_mapping = yamlify_variables(config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping'])
+threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size']
+
+discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout']
+discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval']
+discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout']
+discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries']
+
+network_host = config['configurations']['elastic-site']['network_host']
+network_publish_host = config['configurations']['elastic-site']['network_publish_host']
+
+limits_conf_dir = "/etc/security/limits.d"
+limits_conf_file = limits_conf_dir + "/elasticsearch.conf"
+elastic_user_nofile_limit = config['configurations']['elastic-env']['elastic_user_nofile_limit']
+elastic_user_nproc_limit = config['configurations']['elastic-env']['elastic_user_nproc_limit']
+elastic_user_memlock_soft_limit = config['configurations']['elastic-env']['elastic_user_memlock_soft_limit']
+elastic_user_memlock_hard_limit = config['configurations']['elastic-env']['elastic_user_memlock_hard_limit']
+
+# the status check (service elasticsearch status) cannot be run by the 'elasticsearch'
+# user due to the default permissions that are set when the package is installed. the
+# status check must be run as root
+elastic_status_check_user = 'root'
+
+# when using the RPM or Debian packages on systems that use systemd, system limits
+# must be specified via systemd.
+# see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setting-system-settings.html#systemd
+systemd_parent_dir = '/etc/systemd/system/'
+systemd_elasticsearch_dir = systemd_parent_dir + 'elasticsearch.service.d/'
+systemd_override_file = systemd_elasticsearch_dir + 'override.conf'
+systemd_override_template = config['configurations']['elastic-systemd']['content']
+
+heap_size = config['configurations']['elastic-jvm-options']['heap_size']
+jvm_options_template = config['configurations']['elastic-jvm-options']['content']
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py
new file mode 100755
index 0000000..ef9f6dd
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+
+
+def properties_inline_template(configurations):
+ return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }}
+{% endfor %}''', configurations_dict=configurations)
+
+
+def properties_config(filename, configurations=None, conf_dir=None,
+ mode=None, owner=None, group=None, brokerid=None):
+ config_content = properties_inline_template(configurations)
+ File(format("{conf_dir}/{filename}"), content=config_content, owner=owner,
+ group=group, mode=mode)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
new file mode 100755
index 0000000..3ac7c83
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+import subprocess
+import sys
+import re
+
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.script import Script
+from resource_management.core.logger import Logger
+
+class ServiceCheck(Script):
+
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ Logger.info("Running Elasticsearch service check")
+
+ port = self.get_port_from_range(params.http_port)
+ self.check_cluster_health(params.hostname, port)
+ self.index_document(params.hostname, port)
+
+ Logger.info("Elasticsearch service check successful")
+ exit(0)
+
+ def index_document(self, host, port, doc='{"name": "Ambari Service Check"}', index="ambari_service_check"):
+ """
+ Tests the health of Elasticsearch by indexing a document.
+
+ :param host: The name of a host running Elasticsearch.
+ :param port: The Elasticsearch HTTP port.
+ :param doc: The test document to put.
+ :param index: The name of the test index.
+ """
+ # put a document into a new index
+ Execute("curl -XPUT 'http://%s:%s/%s/test/1' -d '%s'" % (host, port, index, doc), logoutput=True)
+
+ # retrieve the document... use subprocess because we actually need the results here.
+ cmd_retrieve = "curl -XGET 'http://%s:%s/%s/test/1'" % (host, port, index)
+ proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+ response_retrieve = stdout
+ Logger.info("Retrieval response is: %s" % response_retrieve)
+ expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \
+ % (index, doc)
+
+ # delete the test index
+ cmd_delete = "curl -XDELETE 'http://%s:%s/%s'" % (host, port, index)
+ proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+ response_delete = stdout
+ Logger.info("Delete index response is: %s" % response_retrieve)
+ expected_delete = '{"acknowledged":true}'
+
+ if (expected_retrieve == response_retrieve) and (expected_delete == response_delete):
+ Logger.info("Successfully indexed document in Elasticsearch")
+ else:
+ Logger.info("Unable to retrieve document from Elasticsearch")
+ sys.exit(1)
+
+ def check_cluster_health(self, host, port, status="green", timeout="120s"):
+ """
+ Checks Elasticsearch cluster health. Will wait for a given health
+ state to be reached.
+
+ :param host: The name of a host running Elasticsearch.
+ :param port: The Elasticsearch HTTP port.
+ :param status: The expected cluster health state. By default, green.
+ :param timeout: How long to wait for the cluster. By default, 120 seconds.
+ """
+ Logger.info("Checking cluster health")
+
+ cmd = "curl -sS -XGET 'http://{0}:{1}/_cluster/health?wait_for_status={2}&timeout={3}' | grep '\"status\":\"{2}\"'"
+ Execute(cmd.format(host, port, status, timeout), logoutput=True, tries=5, try_sleep=10)
+
+ def get_port_from_range(self, port_range, delimiter="-", default="9200"):
+ """
+ Elasticsearch is configured with a range of ports to bind to, such as
+ 9200-9300. This function identifies a single port within the given range.
+
+ :param port_range: A range of ports that Elasticsearch binds to.
+ :param delimiter: The port range delimiter, by default "-".
+ :param default: If no port can be identified in the port_range, the default is returned.
+ :return A single port within the given range.
+ """
+ port = default
+ if delimiter in port_range:
+ ports = port_range.split(delimiter)
+ if len(ports) > 0:
+ port = ports[0]
+
+ return port
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
new file mode 100755
index 0000000..0629735
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+
+config = Script.get_config()
+
+elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
+elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid")
+elastic_user = config['configurations']['elastic-env']['elastic_user']
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2
new file mode 100755
index 0000000..8e20ba2
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2
@@ -0,0 +1,77 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+cluster:
+ name: {{cluster_name}}
+ routing:
+ allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
+ allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
+ allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
+ allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
+
+discovery:
+ zen:
+ ping:
+ unicast:
+ hosts: {{zen_discovery_ping_unicast_hosts}}
+
+node:
+ data: {{ masters_also_are_datanodes }}
+ master: true
+ name: {{hostname}}
+path:
+ data: {{path_data}}
+
+http:
+ port: {{http_port}}
+ cors.enabled: {{http_cors_enabled}}
+
+
+transport:
+ tcp:
+ port: {{transport_tcp_port}}
+
+gateway:
+ recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
+ recover_after_time: {{recover_after_time}}
+ expected_data_nodes: {{expected_data_nodes}}
+# https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html
+indices:
+ store.throttle.type: {{indices_memory_index_store_throttle_type}}
+ memory:
+ index_buffer_size: {{indices_memory_index_buffer_size}}
+ fielddata:
+ cache.size: {{indices_fielddata_cache_size}}
+
+bootstrap:
+ memory_lock: {{bootstrap_memory_lock}}
+ system_call_filter: false
+
+thread_pool:
+ bulk:
+ queue_size: {{threadpool_bulk_queue_size}}
+ index:
+ queue_size: {{threadpool_index_queue_size}}
+
+discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
+discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
+discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
+discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
+
+network.host: {{network_host}}
+network.publish_host: {{network_publish_host}}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2
new file mode 100755
index 0000000..6bf8399
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2
@@ -0,0 +1,78 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+cluster:
+ name: {{cluster_name}}
+ routing:
+ allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
+ allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
+ allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
+ allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
+
+discovery:
+ zen:
+ ping:
+ unicast:
+ hosts: {{zen_discovery_ping_unicast_hosts}}
+
+node:
+ data: true
+ master: false
+ name: {{hostname}}
+path:
+ data: {{path_data}}
+
+http:
+ port: {{http_port}}
+ cors.enabled: {{http_cors_enabled}}
+
+
+transport:
+ tcp:
+ port: {{transport_tcp_port}}
+
+gateway:
+ recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
+ recover_after_time: {{recover_after_time}}
+ expected_data_nodes: {{expected_data_nodes}}
+
+# https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html
+indices:
+ store.throttle.type: {{indices_memory_index_store_throttle_type}}
+ memory:
+ index_buffer_size: {{indices_memory_index_buffer_size}}
+ fielddata:
+ cache.size: {{indices_fielddata_cache_size}}
+
+bootstrap:
+ memory_lock: {{bootstrap_memory_lock}}
+ system_call_filter: false
+
+thread_pool:
+ bulk:
+ queue_size: {{threadpool_bulk_queue_size}}
+ index:
+ queue_size: {{threadpool_index_queue_size}}
+
+discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
+discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
+discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
+discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
+
+network.host: {{network_host}}
+network.publish_host: {{network_publish_host}}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2
new file mode 100644
index 0000000..99f72e1
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{elastic_user}} - nproc {{elastic_user_nproc_limit}}
+{{elastic_user}} - nofile {{elastic_user_nofile_limit}}
+{{elastic_user}} soft memlock {{elastic_user_memlock_soft_limit}}
+{{elastic_user}} hard memlock {{elastic_user_memlock_hard_limit}}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json
new file mode 100644
index 0000000..909828b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json
@@ -0,0 +1,43 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "es_health_link",
+ "label": "Elasticsearch Health",
+ "requires_user_name": "false",
+ "component_name": "ES_MASTER",
+ "url":"%@://%@:%@/_cat/health?v",
+ "port":{
+ "http_property": "http_port",
+ "http_default_port": "9200",
+ "https_property": "http_port",
+ "https_default_port": "9200",
+ "regex": "^(\\d+)",
+ "site": "elastic-site"
+ }
+ },
+ {
+ "name": "es_indices_link",
+ "label": "Elasticsearch Indexes",
+ "requires_user_name": "false",
+ "component_name": "ES_MASTER",
+ "url":"%@://%@:%@/_cat/indices?v",
+ "port":{
+ "http_property": "http_port",
+ "http_default_port": "9200",
+ "https_property": "http_port",
+ "https_default_port": "9200",
+ "regex": "^(\\d+)",
+ "site": "elastic-site"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json
new file mode 100755
index 0000000..130d018
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json
@@ -0,0 +1,8 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml
new file mode 100755
index 0000000..1246405
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>kibana_user</name>
+ <value>kibana</value>
+ <property-type>USER</property-type>
+ <description>Service User for Kibana</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ </property>
+ <property>
+ <name>kabana_group</name>
+ <value>kibana</value>
+ <property-type>GROUP</property-type>
+ <description>Service Group for Kibana</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ </property>
+ <property require-input="true">
+ <name>kibana_server_host</name>
+ <value>0.0.0.0</value>
+ <description>Host name or IP address that Kibana should bind to.</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_log_dir</name>
+ <value>/var/log/kibana</value>
+ <description>Log directory for Kibana</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_pid_dir</name>
+ <value>/var/run/kibana</value>
+ <description>PID directory for Kibana</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_es_url</name>
+ <value></value>
+ <description>The Elasticsearch instance to use for all your queries. (http://eshost:9200)</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_server_port</name>
+ <value>5000</value>
+ <description>Kibana back end server port to use.</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_default_application</name>
+ <value>default</value>
+ <description>The default application to load.</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml
new file mode 100755
index 0000000..d8d0513
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <!-- kibana.yml -->
+ <property>
+ <name>content</name>
+ <display-name>kibana.yml template</display-name>
+ <description>This is the jinja template for kibana.yml file</description>
+ <value>
+# Kibana is served by a back end server. This controls which port to use.
+server.port: {{ kibana_port }}
+
+# The host to bind the server to.
+# Kibana (like Elasticsearch) now binds to localhost for security purposes instead of 0.0.0.0 (all addresses). Previous binding to 0.0.0.0 also caused issues for Windows users.
+server.host: {{ kibana_server_host }}
+
+# If you are running kibana behind a proxy, and want to mount it at a path,
+# specify that path here. The basePath can't end in a slash.
+# server.basePath: ""
+
+# The maximum payload size in bytes on incoming server requests.
+# server.maxPayloadBytes: 1048576
+
+# The Elasticsearch instance to use for all your queries.
+elasticsearch.url: {{ es_url }}
+
+# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
+# then the host you use to connect to *this* Kibana instance will be sent.
+# elasticsearch.preserveHost: true
+
+# Kibana uses an index in Elasticsearch to store saved searches, visualizations
+# and dashboards. It will create a new index if it doesn't already exist.
+# kibana.index: ".kibana"
+
+# The default application to load.
+kibana.defaultAppId: "{{ kibana_default_application }}"
+
+# If your Elasticsearch is protected with basic auth, these are the user credentials
+# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana
+# users will still need to authenticate with Elasticsearch (which is proxied through
+# the Kibana server)
+# elasticsearch.username: "user"
+# elasticsearch.password: "pass"
+
+# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted)
+# server.ssl.cert: /path/to/your/server.crt
+# server.ssl.key: /path/to/your/server.key
+
+# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted)
+# elasticsearch.ssl.cert: /path/to/your/client.crt
+# elasticsearch.ssl.key: /path/to/your/client.key
+
+# If you need to provide a CA certificate for your Elasticsearch instance, put
+# the path of the pem file here.
+# elasticsearch.ssl.ca: /path/to/your/CA.pem
+
+# Set to false to have a complete disregard for the validity of the SSL
+# certificate.
+# elasticsearch.ssl.verify: true
+
+# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to
+# request_timeout setting
+# elasticsearch.pingTimeout: 1500
+
+# Time in milliseconds to wait for responses from the back end or elasticsearch.
+# This must be > 0
+# elasticsearch.requestTimeout: 30000
+
+# Time in milliseconds for Elasticsearch to wait for responses from shards.
+# Set to 0 to disable.
+# elasticsearch.shardTimeout: 0
+
+# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying
+# elasticsearch.startupTimeout: 5000
+
+# Set the path to where you would like the process id file to be created.
+# pid.file: /var/run/kibana.pid
+
+# If you would like to send the log output to a file you can set the path below.
+logging.dest: {{ log_dir }}/kibana.log
+
+# Set this to true to suppress all logging output.
+# logging.silent: false
+
+# Set this to true to suppress all logging output except for error messages.
+# logging.quiet: false
+
+# Set this to true to log all events, including system usage information and all requests.
+# logging.verbose: false
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
new file mode 100755
index 0000000..b542c54
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>KIBANA</name>
+ <displayName>Kibana</displayName>
+ <comment>Kibana Dashboard</comment>
+ <version>5.6.2</version>
+ <components>
+ <component>
+ <name>KIBANA_MASTER</name>
+ <displayName>Kibana Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <commandScript>
+ <script>scripts/kibana_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat6</osFamily>
+ <packages>
+ <package>
+ <name>python-elasticsearch</name>
+ </package>
+ <package>
+ <name>kibana-5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat7</osFamily>
+ <packages>
+ <package>
+ <name>python-elasticsearch</name>
+ </package>
+ <package>
+ <name>kibana-5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>ubuntu14</osFamily>
+ <packages>
+ <package>
+ <name>kibana=5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <configuration-dependencies>
+ <config-type>kibana-env</config-type>
+ <config-type>kibana-site</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
new file mode 100644
index 0000000..37100cd
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
@@ -0,0 +1,56 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.get_user_call_output import get_user_call_output
+from resource_management.core.exceptions import ExecutionFailed
+from resource_management.core.exceptions import ComponentIsNotRunning
+
+def service_check(cmd, user, label):
+ """
+ Executes a service check command that adheres to LSB-compliant
+ return codes. The return codes are interpreted as defined
+ by the LSB.
+
+ See http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
+ for more information.
+
+ :param cmd: The service check command to execute.
+ :param label: The name of the service.
+ """
+ Logger.info("Performing service check; cmd={0}, user={1}, label={2}".format(cmd, user, label))
+ rc, out, err = get_user_call_output(cmd, user, is_checked_call=False)
+
+ if len(err) > 0:
+ Logger.error(err)
+
+ if rc in [1, 2, 3]:
+ # if return code in [1, 2, 3], then 'program is not running' or 'program is dead'
+ Logger.info("{0} is not running".format(label))
+ raise ComponentIsNotRunning()
+
+ elif rc == 0:
+ # if return code = 0, then 'program is running or service is OK'
+ Logger.info("{0} is running".format(label))
+
+ else:
+ # else service state is unknown
+ err_msg = "{0} service check failed; cmd '{1}' returned {2}".format(label, cmd, rc)
+ Logger.error(err_msg)
+ raise ExecutionFailed(err_msg, rc, out, err)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
new file mode 100755
index 0000000..c013ea3
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
@@ -0,0 +1,81 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions.format import format as ambari_format
+from resource_management.libraries.script import Script
+
+from common import service_check
+
+class Kibana(Script):
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+ Logger.info("Installing Kibana")
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+ Logger.info("Configuring Kibana")
+
+ directories = [params.log_dir, params.pid_dir, params.conf_dir]
+ Directory(directories,
+ mode=0755,
+ owner=params.kibana_user,
+ group=params.kibana_user
+ )
+
+ File("{0}/kibana.yml".format(params.conf_dir),
+ owner=params.kibana_user,
+ content=InlineTemplate(params.kibana_yml_template)
+ )
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ Logger.info("Stopping Kibana")
+ Execute("service kibana stop")
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ Logger.info("Starting Kibana")
+ Execute("service kibana start")
+
+ def restart(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ Logger.info("Restarting Kibana")
+ Execute("service kibana restart")
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ Logger.info('Status check Kibana')
+ service_check("service kibana status", user=params.kibana_user, label="Kibana")
+
+if __name__ == "__main__":
+ Kibana().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
new file mode 100755
index 0000000..ef4cb62
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Kibana Params configurations
+
+"""
+
+from urlparse import urlparse
+
+from resource_management.libraries.functions import format
+from resource_management.libraries.script import Script
+
+# server configurations
+config = Script.get_config()
+
+kibana_home = '/usr/share/kibana/'
+kibana_bin = '/usr/share/kibana/bin/'
+
+conf_dir = "/etc/kibana"
+kibana_user = config['configurations']['kibana-env']['kibana_user']
+kibana_group = config['configurations']['kibana-env']['kibana_group']
+log_dir = config['configurations']['kibana-env']['kibana_log_dir']
+pid_dir = config['configurations']['kibana-env']['kibana_pid_dir']
+pid_file = format("{pid_dir}/kibanasearch.pid")
+es_url = config['configurations']['kibana-env']['kibana_es_url']
+parsed = urlparse(es_url)
+es_host = parsed.netloc.split(':')[0]
+es_port = parsed.netloc.split(':')[1]
+kibana_port = config['configurations']['kibana-env']['kibana_server_port']
+kibana_server_host = config['configurations']['kibana-env']['kibana_server_host']
+kibana_default_application = config['configurations']['kibana-env']['kibana_default_application']
+hostname = config['hostname']
+java64_home = config['hostLevelParams']['java_home']
+kibana_yml_template = config['configurations']['kibana-site']['content']
+
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
new file mode 100755
index 0000000..448e102
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "metron_ui",
+ "label": "Metron UI",
+ "requires_user_name": "false",
+ "component_name": "KIBANA_MASTER",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "kibana_server_port",
+ "http_default_port": "5601",
+ "https_property": "kibana_server_port",
+ "https_default_port": "5601",
+ "regex": "^(\\d+)$",
+ "site": "kibana-env"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json
new file mode 100644
index 0000000..1b5400c
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json
@@ -0,0 +1,76 @@
+{
+ "type": "full-release",
+ "name": "elasticsearch-ambari.mpack",
+ "version": "5.6.2",
+ "description": "Ambari Management Pack for Elasticsearch and Kibana",
+ "prerequisites": {
+ "min-ambari-version": "2.4.0.0",
+ "min-stack-versions": [
+ {
+ "stack_name": "HDP",
+ "stack_version": "2.3.0"
+ }
+ ]
+ },
+ "artifacts": [
+ {
+ "name": "ELASTICSEARCH-common-services",
+ "type" : "service-definitions",
+ "source_dir" : "common-services"
+ },
+ {
+ "name" : "ELASTICSEARCH-addon-services",
+ "type" : "stack-addon-service-definitions",
+ "source_dir": "addon-services",
+ "service_versions_map": [
+ {
+ "service_name" : "KIBANA",
+ "service_version" : "5.6.2",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.3"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.4"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.5"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.6"
+ }
+
+ ]
+ },
+ {
+ "service_name" : "ELASTICSEARCH",
+ "service_version" : "5.6.2",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.3"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.4"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.5"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.6"
+ }
+
+
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/README.md b/metron-deployment/packaging/ambari/metron-mpack/README.md
index 5179b5f..cd9399d 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/README.md
+++ b/metron-deployment/packaging/ambari/metron-mpack/README.md
@@ -86,15 +86,11 @@ Storm (and the Metron topologies) must be restarted after Metron is installed on
Kerberizing a cluster with a pre-existing Metron, automatically restarts all services during Kerberization. No additional manual restart is needed in this case.
-#### Zeppelin Import
-
-A custom action is available in Ambari to import Zeppelin dashboards. See the [metron-indexing documentation](../../../../metron-platform/metron-indexing) for more information.
-
#### Kibana Dashboards
The dashboards installed by the Kibana custom action are managed by two JSON files:
-* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
-* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
+* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
+* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json
The first file, `kibana.template`, is an Elasticsearch template that specifies the proper mapping types for the Kibana index. This configuration is necessary due to a bug
in the default dynamic mappings provided by Elasticsearch for long types versus integer that are incompatible with Kibana \[1\]. The second file, `dashboard-bulkload.json`,
@@ -102,7 +98,7 @@ contains all of the dashboard metadata necessary to create the Metron dashboard.
of documents necessary for setting up the dashboard in Elasticsearch. The main features installed are index patterns, searches, and a variety of visualizations
that are used in the Metron dashboard.
-Deploying the existing dashboard is easy. Once the MPack is installed, run the Kibana service's action "Load Template" to install dashboards. This will no longer overwrite
+Deploying the existing dashboard is easy. Once the MPack is installed, run the Metron service's action "Load Template" to install dashboards. This will no longer overwrite
the .kibana in Elasticsearch. The bulk load is configured to fail inserts for existing documents. If you want to _completely_ reload the dashboard, you would need to delete
the .kibana index and reload again from Ambari.
@@ -115,7 +111,7 @@ You can modify dashboards in Kibana and bring those changes into the core MPack
1. Export the .kibana index from ES
2. Convert the data into the ES bulk load format
-3. Replace the dashboard-bulkload.json file in the Kibana MPack.
+3. Replace the dashboard-bulkload.json file in the Metron MPack.
You can export the .kibana index using a tool like [https://github.com/taskrabbit/elasticsearch-dump](https://github.com/taskrabbit/elasticsearch-dump). The important
feature is to have one document per line. Here's an exmaple export using elasticsearch-dump
@@ -166,14 +162,14 @@ To create a new version of the file, make any necessary changes to Kibana (e.g.
**Saving a Backup**
```
-python packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py \
+python packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py \
$ES_HOST 9200 \
~/dashboard.p -s
```
**Restoring From a Backup**
```
-python packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py \
+python packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py \
$ES_HOST 9200 \
~/dashboard.p
```
@@ -181,6 +177,10 @@ $ES_HOST 9200 \
**Note**: This method of writing the Kibana dashboard to Elasticsearch will overwrite the entire .kibana index. Be sure to first backup the index first using either the new JSON
method described above, or writing out the dashboard.p pickle file using the old method (passing -s option to dashboardindex.py) described here.
+#### Zeppelin Import
+
+A custom action is available in Ambari to import Zeppelin dashboards. See the [metron-indexing documentation](../../../../metron-platform/metron-indexing) for more information.
+
#### Offline Installation
Retrieval of the GeoIP database is the only point during installation that reaches out to the internet. For an offline installation, the URL for the GeoIP database can be manually set to a local path on the file system such as `file:///home/root/geoip/GeoLite2-City.mmdb.gz`.
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
index ec4272e..81087a2 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
@@ -30,24 +30,10 @@
<filtered>true</filtered>
</fileSet>
<fileSet>
- <directory>src/main/resources/common-services</directory>
- <outputDirectory>common-services</outputDirectory>
- <excludes>
- <exclude>**/CURRENT/**</exclude>
- </excludes>
- </fileSet>
- <fileSet>
<directory>src/main/resources/addon-services/METRON/CURRENT</directory>
<outputDirectory>addon-services/METRON/${metron.version}</outputDirectory>
<filtered>true</filtered>
</fileSet>
- <fileSet>
- <directory>src/main/resources/addon-services</directory>
- <outputDirectory>addon-services</outputDirectory>
- <excludes>
- <exclude>**/CURRENT/**</exclude>
- </excludes>
- </fileSet>
</fileSets>
<files>
<file>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml
deleted file mode 100755
index accf7da..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>ELASTICSEARCH</name>
- <version>5.6.2</version>
- <extends>common-services/ELASTICSEARCH/5.6.2</extends>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
deleted file mode 100644
index ba21fb1..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<reposinfo>
- <os family="redhat6">
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
- <repoid>elasticsearch-5.x</repoid>
- <reponame>ELASTICSEARCH</reponame>
- </repo>
- </os>
- <os family="redhat7">
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
- <repoid>elasticsearch-5.x</repoid>
- <reponame>ELASTICSEARCH</reponame>
- </repo>
- </os>
- <os family="ubuntu14">
- <!--
- see details about Ambari fixes for Ubuntu introduced in Ambari 2.6+
- https://github.com/apache/ambari/commit/f8b29df9685b443d4a5c06c6e1725e4428c95b49#diff-6f26c26ed59462200d018c5e1e71e773
- https://issues.apache.org/jira/browse/AMBARI-21856
- -->
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/apt</baseurl>
- <repoid>elasticsearch-5.x</repoid>
- <reponame>ELASTICSEARCH</reponame>
- <distribution>stable</distribution>
- </repo>
- </os>
-</reposinfo>
-
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml
deleted file mode 100755
index 8a4fba2..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0"?>
-<!--
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>KIBANA</name>
- <version>5.6.2</version>
- <extends>common-services/KIBANA/5.6.2</extends>
- </service>
- </services>
-</metainfo>
-
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json
deleted file mode 100755
index 622a512..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "kibana_master_ui",
- "label": "Metron Dashboard",
- "requires_user_name": "false",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "kibana_server_port",
- "http_default_port": "5601",
- "https_property": "kibana_server_port",
- "https_default_port": "5601",
- "regex": "^(\\d+)$",
- "site": "kibana-env"
- }
- }
- ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
deleted file mode 100644
index 2755818..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
+++ /dev/null
@@ -1,60 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<reposinfo>
- <os family="redhat6">
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
- <repoid>kibana-5.x</repoid>
- <reponame>KIBANA</reponame>
- </repo>
- <repo>
- <baseurl>http://packages.elastic.co/curator/5/centos/6</baseurl>
- <repoid>ES-Curator-5.x</repoid>
- <reponame>CURATOR</reponame>
- </repo>
- </os>
- <os family="redhat7">
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
- <repoid>kibana-5.x</repoid>
- <reponame>KIBANA</reponame>
- </repo>
- <repo>
- <baseurl>http://packages.elastic.co/curator/5/centos/7</baseurl>
- <repoid>ES-Curator-5.x</repoid>
- <reponame>CURATOR</reponame>
- </repo>
- </os>
- <os family="ubuntu14">
- <!--
- see details about Ambari fixes for Ubuntu introduced in Ambari 2.6+
- https://github.com/apache/ambari/commit/f8b29df9685b443d4a5c06c6e1725e4428c95b49#diff-6f26c26ed59462200d018c5e1e71e773
- https://issues.apache.org/jira/browse/AMBARI-21856
- -->
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/apt</baseurl>
- <repoid>kibana-5.x</repoid>
- <reponame>KIBANA</reponame>
- <distribution>stable</distribution>
- </repo>
- <repo>
- <baseurl>https://packages.elastic.co/curator/5/debian</baseurl>
- <repoid>ES-Curator-5.x</repoid>
- <reponame>CURATOR</reponame>
- <distribution>stable</distribution>
- </repo>
- </os>
-</reposinfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml
deleted file mode 100755
index 9e4f8ad..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml
+++ /dev/null
@@ -1,86 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>elastic_user</name>
- <value>elasticsearch</value>
- <property-type>USER</property-type>
- <description>Service user for Elasticsearch</description>
- <value-attributes>
- <type>user</type>
- <overridable>false</overridable>
- </value-attributes>
- </property>
- <property>
- <name>elastic_group</name>
- <value>elasticsearch</value>
- <property-type>GROUP</property-type>
- <description>Service group for Elasticsearch</description>
- <value-attributes>
- <type>user</type>
- <overridable>false</overridable>
- </value-attributes>
- </property>
- <property>
- <name>elastic_log_dir</name>
- <value>/var/log/elasticsearch</value>
- <description>Log directory for elastic</description>
- </property>
- <property>
- <name>elastic_pid_dir</name>
- <value>/var/run/elasticsearch</value>
- <description>The directory for pid files</description>
- </property>
- <!-- elasticsearch-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for elastic-env.sh file</description>
- <value>
-#!/bin/bash
-
-# Set ELASTICSEARCH specific environment variables here.
-
-# The java implementation to use.
-export JAVA_HOME={{java64_home}}
-export PATH=$PATH:$JAVA_HOME/bin
- </value>
- </property>
- <property>
- <name>elastic_user_nofile_limit</name>
- <value>65536</value>
- <description>Max open file limit for Elasticsearch user.</description>
- </property>
- <property>
- <name>elastic_user_nproc_limit</name>
- <value>2048</value>
- <description>Max number of processes for Elasticsearch user.</description>
- </property>
- <property>
- <name>elastic_user_memlock_soft_limit</name>
- <value>unlimited</value>
- <description>Max locked-in memory address space (soft memlock limit).</description>
- </property>
- <property>
- <name>elastic_user_memlock_hard_limit</name>
- <value>unlimited</value>
- <description>Max locked-in memory address space (hard memlock limit).</description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
deleted file mode 100644
index 5c6aaca..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
+++ /dev/null
@@ -1,144 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>heap_size</name>
- <value>512m</value>
- <description>JVM heap size</description>
- </property>
- <property>
- <name>content</name>
- <description>The jinja template for the Elasticsearch JVM options file.</description>
- <value>
-## JVM configuration
-
-################################################################
-## IMPORTANT: JVM heap size
-################################################################
-##
-## You should always set the min and max JVM heap
-## size to the same value. For example, to set
-## the heap to 4 GB, set:
-##
-## -Xms4g
-## -Xmx4g
-##
-## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
-## for more information
-##
-################################################################
-
-# Xms represents the initial size of total heap space
-# Xmx represents the maximum size of total heap space
-
--Xms{{heap_size}}
--Xmx{{heap_size}}
-
-################################################################
-## Expert settings
-################################################################
-##
-## All settings below this section are considered
-## expert settings. Don't tamper with them unless
-## you understand what you are doing
-##
-################################################################
-
-## GC configuration
--XX:+UseConcMarkSweepGC
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
-
-## optimizations
-
-# pre-touch memory pages used by the JVM during initialization
--XX:+AlwaysPreTouch
-
-## basic
-
-# force the server VM (remove on 32-bit client JVMs)
--server
-
-# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
--Xss1m
-
-# set to headless, just in case
--Djava.awt.headless=true
-
-# ensure UTF-8 encoding by default (e.g. filenames)
--Dfile.encoding=UTF-8
-
-# use our provided JNA always versus the system one
--Djna.nosys=true
-
-# use old-style file permissions on JDK9
--Djdk.io.permissionsUseCanonicalPath=true
-
-# flags to configure Netty
--Dio.netty.noUnsafe=true
--Dio.netty.noKeySetOptimization=true
--Dio.netty.recycler.maxCapacityPerThread=0
-
-# log4j 2
--Dlog4j.shutdownHookEnabled=false
--Dlog4j2.disable.jmx=true
--Dlog4j.skipJansi=true
-
-## heap dumps
-
-# generate a heap dump when an allocation from the Java heap fails
-# heap dumps are created in the working directory of the JVM
--XX:+HeapDumpOnOutOfMemoryError
-
-# specify an alternative path for heap dumps
-# ensure the directory exists and has sufficient space
-#-XX:HeapDumpPath=${heap.dump.path}
-
-## GC logging
-
-#-XX:+PrintGCDetails
-#-XX:+PrintGCTimeStamps
-#-XX:+PrintGCDateStamps
-#-XX:+PrintClassHistogram
-#-XX:+PrintTenuringDistribution
-#-XX:+PrintGCApplicationStoppedTime
-
-# log GC status to a file with time stamps
-# ensure the directory exists
-#-Xloggc:${loggc}
-
-# By default, the GC log file will not rotate.
-# By uncommenting the lines below, the GC log file
-# will be rotated every 128MB at most 32 times.
-#-XX:+UseGCLogFileRotation
-#-XX:NumberOfGCLogFiles=32
-#-XX:GCLogFileSize=128M
-
-# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
-# If documents were already indexed with unquoted fields in a previous version
-# of Elasticsearch, some operations may throw errors.
-#
-# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
-# only for migration purposes.
-#-Delasticsearch.json.allow_unquoted_field_names=true
- </value>
- </property>
-</configuration>