You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2016/09/26 14:05:07 UTC
[4/7] incubator-metron git commit: METRON-427 Create Ambari
Management Pack for Metron Installation closes apache/incubator-metron#266
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py
new file mode 100755
index 0000000..b3fb809
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import subprocess
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class ParserCommands:
+ __params = None
+ __parser_list = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__parser_list = self.__get_parsers(params)
+ self.__configured = os.path.isfile(self.__params.parsers_configured_flag_file)
+
+ # get list of parsers
+ def __get_parsers(self, params):
+ return params.parsers.replace(' ', '').split(',')
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.parsers_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def init_parsers(self):
+ Logger.info(
+ "Copying grok patterns from local directory '{0}' to HDFS '{1}'".format(self.__params.local_grok_patterns_dir,
+ self.__params.metron_apps_dir))
+ self.__params.HdfsResource(self.__params.metron_apps_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ mode=0775,
+ source=self.__params.local_grok_patterns_dir)
+
+ Logger.info("Done initializing parser configuration")
+
+ def get_parser_list(self):
+ return self.__parser_list
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.0BETA]\n"
+ "name=Metron 0.2.0BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating main topics for parsers")
+ for parser_name in self.get_parser_list():
+ Logger.info("Creating topic'{0}'".format(parser_name))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ parser_name,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Creating topics for error handling")
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_invalid",
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_error",
+ num_partitions, replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_parser_topologies(self):
+ Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list()))
+ start_cmd_template = """{0}/bin/start_parser_topology.sh \
+ -k {1} \
+ -z {2} \
+ -s {3}"""
+ for parser in self.get_parser_list():
+ Logger.info('Starting ' + parser)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
+ self.__params.zookeeper_quorum, parser))
+
+ Logger.info('Finished starting parser topologies')
+
+ def stop_parser_topologies(self):
+ Logger.info('Stopping parsers')
+ for parser in self.get_parser_list():
+ Logger.info('Stopping ' + parser)
+ stop_cmd = 'storm kill ' + parser
+ Execute(stop_cmd)
+ Logger.info('Done stopping parser topologies')
+
+ def restart_parser_topologies(self, env):
+ Logger.info('Restarting the parser topologies')
+ self.stop_parser_topologies()
+ attempt_count = 0
+ while self.topologies_running(env):
+ if attempt_count > 2:
+ raise Exception("Unable to kill topologies")
+ attempt_count += 1
+ time.sleep(10)
+ self.start_parser_topologies()
+ Logger.info('Done restarting the parser topologies')
+
+ def topologies_exist(self):
+ cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (stdout, stderr) = cmd_open.communicate()
+ stdout_lines = stdout.splitlines()
+ if stdout_lines:
+ status_lines = self.__get_status_lines(stdout_lines)
+ for parser in self.get_parser_list():
+ for line in status_lines:
+ items = re.sub('[\s]+', ' ', line).split()
+ if items and items[0] == parser:
+ return True
+ return False
+
+ def topologies_running(self, env):
+ env.set_params(self.__params)
+ all_running = True
+ topologies = metron_service.get_running_topologies()
+ for parser in self.get_parser_list():
+ parser_found = False
+ is_running = False
+ if parser in topologies:
+ parser_found = True
+ is_running = topologies[parser] in ['ACTIVE', 'REBALANCING']
+ all_running &= parser_found and is_running
+ return all_running
+
+ def __get_status_lines(self, lines):
+ status_lines = []
+ do_stat = False
+ skipped = 0
+ for line in lines:
+ if line.startswith("Topology_name"):
+ do_stat = True
+ if do_stat and skipped == 2:
+ status_lines += [line]
+ elif do_stat:
+ skipped += 1
+ return status_lines
+
+ def __is_running(self, status):
+ return status in ['ACTIVE', 'REBALANCING']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py
new file mode 100755
index 0000000..3758873
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py
@@ -0,0 +1,89 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.libraries.script import Script
+
+import metron_service
+from parser_commands import ParserCommands
+
+
+class ParserMaster(Script):
+ def get_component_name(self):
+ # TODO add this at some point - currently will cause problems with hdp-select
+ # return "parser-master"
+ pass
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = ParserCommands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+ metron_service.load_global_config(params)
+ commands = ParserCommands(params)
+ if not commands.is_configured():
+ commands.init_parsers()
+ commands.init_kafka_topics()
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = ParserCommands(params)
+ commands.start_parser_topologies()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = ParserCommands(params)
+ commands.stop_parser_topologies()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = ParserCommands(status_params)
+ if not commands.topologies_running(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = ParserCommands(params)
+ commands.restart_parser_topologies(env)
+
+ def servicechecktest(self, env):
+ from params import params
+ env.set_params(params)
+ from service_check import ServiceCheck
+ service_check = ServiceCheck()
+ Logger.info('Service Check Test')
+ service_check.service_check(env)
+
+
+if __name__ == "__main__":
+ ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py
new file mode 100755
index 0000000..7dd9dfb
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+from resource_management.libraries.script import Script
+
+from indexing_commands import IndexingCommands
+from parser_commands import ParserCommands
+
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ from params import params
+ parsercommands = ParserCommands(params)
+ indexingcommands = IndexingCommands(params)
+ all_found = parsercommands.topologies_running(env) and indexingcommands.is_topology_active(env)
+ if all_found:
+ exit(0)
+ else:
+ exit(1)
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2
new file mode 100755
index 0000000..bab2f52
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+##### Kafka #####
+
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+enrichment.output.topic=indexing
+
+##### MySQL #####
+
+mysql.ip={{mysql_host}}
+mysql.port={{mysql_port}}
+mysql.username={{enrichment_metron_user}}
+mysql.password={{enrichment_metron_user_passwd}}
+
+##### Metrics #####
+
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+
+#Graphite Addresses
+
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+
+
+#GenericEnrichmentBolt
+org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
+org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
+org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
+
+
+#TelemetryIndexingBolt
+org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
+org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
+org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
+
+##### Host Enrichment #####
+
+hbase.provider.impl=org.apache.metron.hbase.HTableProvider
+enrichment.simple.hbase.table={{enrichment_table}}
+enrichment.simple.hbase.cf={{enrichment_cf}}
+org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+
+##### HBase #####
+bolt.hbase.table.name=pcap
+bolt.hbase.table.fields=t:value
+bolt.hbase.table.key.tuple.field.name=key
+bolt.hbase.table.timestamp.tuple.field.name=timestamp
+bolt.hbase.enable.batching=false
+bolt.hbase.write.buffer.size.in.bytes=2000000
+bolt.hbase.durability=SKIP_WAL
+bolt.hbase.partitioner.region.info.refresh.interval.mins=60
+
+##### Threat Intel #####
+
+threat.intel.tracker.table={{threatintel_table}}
+threat.intel.tracker.cf={{threatintel_cf}}
+threat.intel.simple.hbase.table={{threatintel_table}}
+threat.intel.simple.hbase.cf={{threatintel_cf}}
+threat.intel.ip.table=
+threat.intel.ip.cf=
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json
new file mode 100755
index 0000000..ee1b225
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "storm_ui",
+ "label": "Storm UI",
+ "requires_user_name": "false",
+ "component_name": "STORM_UI_SERVER",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "ui.port",
+ "http_default_port": "8744",
+ "https_property": "ui.port",
+ "https_default_port": "8744",
+ "regex": "^(\\d+)$",
+ "site": "storm-site"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
new file mode 100644
index 0000000..99dec9b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
@@ -0,0 +1,85 @@
+{
+ "type": "full-release",
+ "name": "metron-ambari.mpack",
+ "version": "1.0.0.0",
+ "description": "Ambari Management Pack for Apache Metron",
+ "prerequisites": {
+ "min-ambari-version": "2.4.0.0",
+ "min-stack-versions": [
+ {
+ "stack_name": "HDP",
+ "stack_version": "2.3.0"
+ }
+ ]
+ },
+ "artifacts": [
+ {
+ "name": "METRON-common-services",
+ "type" : "service-definitions",
+ "source_dir" : "common-services"
+ },
+ {
+ "name" : "METRON-addon-services",
+ "type" : "stack-addon-service-definitions",
+ "source_dir": "addon-services",
+ "service_versions_map": [
+ {
+ "service_name" : "KIBANA",
+ "service_version" : "4.5.1",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.3"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.4"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.5"
+ }
+ ]
+ },
+ {
+ "service_name" : "ELASTICSEARCH",
+ "service_version" : "2.3.3",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.3"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.4"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.5"
+ }
+
+ ]
+ },
+ {
+ "service_name" : "METRON",
+ "service_version" : "0.2.0BETA",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.3"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.4"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.5"
+ }
+
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
deleted file mode 100755
index 2939c28..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
+++ /dev/null
@@ -1,58 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>elastic_user</name>
- <value>elasticsearch</value>
- <property-type>USER</property-type>
- <description>The user for Elasticsearch</description>
- </property>
- <property>
- <name>user_group</name>
- <value>elasticsearch</value>
- <description>The group for Elasticsearch</description>
- </property>
- <property>
- <name>elastic_log_dir</name>
- <value>/var/log/elasticsearch</value>
- <description>Log directory for elastic</description>
- </property>
- <property>
- <name>elastic_pid_dir</name>
- <value>/var/run/elasticsearch</value>
- <description>The directory for pid files</description>
- </property>
-
- <!-- elasticsearch-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for elastic-env.sh file</description>
- <value>
-#!/bin/bash
-
-# Set ELASTICSEARCH specific environment variables here.
-
-# The java implementation to use.
-export JAVA_HOME={{java64_home}}
-export PATH=$PATH:$JAVA_HOME/bin
- </value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
deleted file mode 100755
index fb3a443..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
+++ /dev/null
@@ -1,180 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Elastic search Configurations -->
-
-<configuration supports_final="true">
- <!-- Configurations -->
- <property>
- <name>cluster_name</name>
- <value>metron</value>
- <description>Cluster name identifies your cluster</description>
- </property>
- <property>
- <name>zen_discovery_ping_unicast_hosts</name>
- <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.-->
- <value></value>
- <description>Unicast discovery list of hosts to act as gossip routers, in comma separated format.</description>
- </property>
- <property>
- <name>index_number_of_shards</name>
- <value>4</value>
- <description>Set the number of shards (splits) of an index</description>
- </property>
- <property>
- <name>index_number_of_replicas</name>
- <value>2</value>
- <description>Set the number of replicas (additional copies) of an index</description>
- </property>
- <!-- Logging Configurations -->
- <property>
- <name>path_data</name>
- <value>"/opt/lmm/es_data"</value>
- <description>Path to directory where to store index data allocated for this node. e.g. "/mnt/first", "/mnt/second"</description>
- </property>
- <!-- Discovery -->
- <property>
- <name>transport_tcp_port</name>
- <value>9300-9400</value>
- <description>Set a custom port for the node to node communication</description>
- </property>
- <property>
- <name>http_port</name>
- <value>9200-9300</value>
- <description>Set a custom port to listen for HTTP traffic</description>
- </property>
- <!-- Discovery -->
- <property>
- <name>discovery_zen_ping_multicast_enabled</name>
- <value>false</value>
- <description>master eligible nodes</description>
- </property>
- <property>
- <name>discovery_zen_ping_timeout</name>
- <value>3s</value>
- <description>Wait for ping responses for master discovery</description>
- </property>
- <property>
- <name>discovery_zen_fd_ping_interval</name>
- <value>15s</value>
- <description>Wait for ping for cluster discovery</description>
- </property>
- <property>
- <name>discovery_zen_fd_ping_timeout</name>
- <value>60s</value>
- <description>Wait for ping for cluster discovery</description>
- </property>
- <property>
- <name>discovery_zen_fd_ping_retries</name>
- <value>5</value>
- <description>Number of ping retries before blacklisting</description>
- </property>
- <!-- Gateway -->
- <property>
- <name>gateway_recover_after_data_nodes</name>
- <value>3</value>
- <description>Recover as long as this many data or master nodes have joined the cluster.</description>
- </property>
- <property>
- <name>recover_after_time</name>
- <value>15m</value>
- <description>recover_after_time</description>
- </property>
- <property>
- <name>expected_data_nodes</name>
- <value>0</value>
- <description>expected_data_nodes</description>
- </property>
- <!-- Index -->
- <property>
- <name>index_merge_scheduler_max_thread_count</name>
- <value>5</value>
- <description>index.merge.scheduler.max_thread_count</description>
- </property>
- <property>
- <name>indices_memory_index_store_throttle_type</name>
- <value>none</value>
- <description>index_store_throttle_type</description>
- </property>
- <property>
- <name>index_refresh_interval</name>
- <value>1s</value>
- <description>index refresh interval</description>
- </property>
- <property>
- <name>index_translog_flush_threshold_size</name>
- <value>5g</value>
- <description>index_translog_flush_threshold_size</description>
- </property>
- <property>
- <name>indices_memory_index_buffer_size</name>
- <value>10%</value>
- <description>Percentage of heap used for write buffers</description>
- </property>
- <property>
- <name>bootstrap_mlockall</name>
- <value>true</value>
- <description>The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out</description>
- </property>
- <property>
- <name>threadpool_bulk_queue_size</name>
- <value>3000</value>
- <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request</description>
- </property>
- <property>
- <name>threadpool_index_queue_size</name>
- <value>1000</value>
- <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request</description>
- </property>
- <property>
- <name>indices_cluster_send_refresh_mapping</name>
- <value>false</value>
- <description>In order to make the index request more efficient, we have set this property on our data nodes</description>
- </property>
- <property>
- <name>indices_fielddata_cache_size</name>
- <value>25%</value>
- <description>You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index</description>
- </property>
- <property>
- <name>cluster_routing_allocation_disk_watermark_high</name>
- <value>0.99</value>
- <description>Property used when multiple drives are used to understand max thresholds</description>
- </property>
- <property>
- <name>cluster_routing_allocation_disk_threshold_enabled</name>
- <value>true</value>
- <description>Property used when multiple drives are used to understand if thresholding is active</description>
- </property>
- <property>
- <name>cluster_routing_allocation_disk_watermark_low</name>
- <value>.97</value>
- <description>Property used when multiple drives are used to understand min thresholds</description>
- </property>
- <property>
- <name>cluster_routing_allocation_node_concurrent_recoveries</name>
- <value>4</value>
- <description>Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart</description>
- </property>
- <property>
- <name>network_host</name>
- <value>_lo_,_eth0_</value>
- <description>Network interface(s) will bind to. </description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
deleted file mode 100755
index 58e4916..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
+++ /dev/null
@@ -1,98 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>elastic_home</name>
- <value>/usr/share/elasticsearch/</value>
- <description>Elasticsearch Home Directory</description>
- </property>
- <property>
- <name>data_dir</name>
- <value>/var/lib/elasticsearch/</value>
- <description>Elasticsearch Data Directory</description>
- </property>
- <property>
- <name>work_dir</name>
- <value>/tmp/elasticsearch/</value>
- <description>Elasticsearch Work Directory</description>
- </property>
- <property>
- <name>conf_dir</name>
- <value>/etc/elasticsearch/</value>
- <description>Elasticsearch Configuration Directory</description>
- </property>
- <property>
- <name>heap_size</name>
- <value>128m</value>
- <description>Heap size</description>
- </property>
- <property>
- <name>max_open_files</name>
- <value>65535</value>
- <description>Maximum number of open files</description>
- </property>
- <property>
- <name>max_map_count</name>
- <value>262144</value>
- <description>Maximum number of memory map areas for process</description>
- </property>
-
- <!--/etc/sysconfig/elasticsearch-->
- <property>
- <name>content</name>
- <description>This is the jinja template for elastic-env.sh file</description>
- <value>
-# Directory where the Elasticsearch binary distribution resides
-ES_HOME={{elastic_home}}
-
-# Heap Size (defaults to 256m min, 1g max)
-ES_HEAP_SIZE={{heap_size}}
-
-# Maximum number of open files
-MAX_OPEN_FILES={{max_open_files}}
-
-# Maximum number of VMA (Virtual Memory Areas) a process can own
-MAX_MAP_COUNT={{max_map_count}}
-
-# Elasticsearch log directory
-LOG_DIR={{log_dir}}
-
-# Elasticsearch data directory
-DATA_DIR={{data_dir}}
-
-# Elasticsearch work directory
-WORK_DIR={{work_dir}}
-
-# Elasticsearch conf directory
-CONF_DIR={{conf_dir}}
-
-# User to run as, change this to a specific elasticsearch user if possible
-# Also make sure, this user can write into the log directories in case you change them
-# This setting only works for the init script, but has to be configured separately for systemd startup
-ES_USER={{elastic_user}}
-
-# Additional Java OPTS
-ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled
--XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
--XX:ErrorFile={{log_dir}}elasticsearch_err.log -XX:ParallelGCThreads=8"
- </value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
deleted file mode 100755
index a420131..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
+++ /dev/null
@@ -1,73 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>ELASTICSEARCH</name>
- <displayName>Elasticsearch</displayName>
- <comment>Indexing and Search</comment>
- <version>2.3.3</version>
- <components>
- <component>
- <name>ES_MASTER</name>
- <displayName>Elasticsearch Master-Eligible Node</displayName>
- <category>MASTER</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/elastic_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- <component>
- <name>ES_SLAVE</name>
- <displayName>Elasticsearch Data Node</displayName>
- <category>SLAVE</category>
- <cardinality>0+</cardinality>
- <commandScript>
- <script>scripts/elastic_slave.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>elasticsearch-2.3.3</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
- <configuration-dependencies>
- <config-type>elastic-env</config-type>
- <config-type>elastic-site</config-type>
- <config-type>elastic-sysconfig</config-type>
- </configuration-dependencies>
- <restartRequiredAfterChange>true</restartRequiredAfterChange>
- </service>
- </services>
-</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
deleted file mode 100755
index bd858cd..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-from resource_management.core.source import Template
-
-
-def elastic():
- print "INSIDE THE %s" % __file__
- import params
-
- params.path_data = params.path_data.replace('"', '')
- data_path = params.path_data.replace(' ', '').split(',')
- data_path[:] = [x.replace('"', '') for x in data_path]
-
- directories = [params.log_dir, params.pid_dir, params.conf_dir]
- directories = directories + data_path
-
- Directory(directories,
- create_parents=True,
- # recursive=True,
- mode=0755,
- owner=params.elastic_user,
- group=params.elastic_user
- )
-
- print "Master env: ""{}/elastic-env.sh".format(params.conf_dir)
- File("{}/elastic-env.sh".format(params.conf_dir),
- owner=params.elastic_user,
- content=InlineTemplate(params.elastic_env_sh_template)
- )
-
- configurations = params.config['configurations']['elastic-site']
-
- print "Master yml: ""{}/elasticsearch.yml".format(params.conf_dir)
- File("{}/elasticsearch.yml".format(params.conf_dir),
- content=Template(
- "elasticsearch.master.yaml.j2",
- configurations=configurations),
- owner=params.elastic_user,
- group=params.elastic_user
- )
-
- print "Master sysconfig: /etc/sysconfig/elasticsearch"
- File(format("/etc/sysconfig/elasticsearch"),
- owner="root",
- group="root",
- content=InlineTemplate(params.sysconfig_template)
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
deleted file mode 100755
index 5fc29cf..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
+++ /dev/null
@@ -1,79 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from elastic import elastic
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.script import Script
-
-
-class Elasticsearch(Script):
- def install(self, env):
- import params
- env.set_params(params)
-
- print 'Install the Master'
- Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
- Execute("echo \"[elasticsearch-2.x]\n"
- "name=Elasticsearch repository for 2.x packages\n"
- "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n"
- "gpgcheck=1\n"
- "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
- "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo")
-
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- import params
- env.set_params(params)
-
- elastic()
-
- def stop(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- stop_cmd = format("service elasticsearch stop")
- print 'Stop the Master'
- Execute(stop_cmd)
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
-
- self.configure(env)
- start_cmd = format("service elasticsearch start")
- print 'Start the Master'
- Execute(start_cmd)
-
- def status(self, env):
- import params
- env.set_params(params)
- status_cmd = format("service elasticsearch status")
- print 'Status of the Master'
- Execute(status_cmd)
-
- def restart(self, env):
- import params
- env.set_params(params)
- self.configure(env)
- restart_cmd = format("service elasticsearch restart")
- print 'Restarting the Master'
- Execute(restart_cmd)
-
-if __name__ == "__main__":
- Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
deleted file mode 100755
index e65bd8f..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script import Script
-from resource_management.core.resources.system import Execute
-from slave import slave
-
-
-class Elasticsearch(Script):
- def install(self, env):
- import params
- env.set_params(params)
- print 'Install the Slave'
- Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
- Execute("echo \"[elasticsearch-2.x]\n"
- "name=Elasticsearch repository for 2.x packages\n"
- "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n"
- "gpgcheck=1\n"
- "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
- "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo")
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- import params
- env.set_params(params)
- slave()
-
- def stop(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- stop_cmd = format("service elasticsearch stop")
- print 'Stop the Slave'
- Execute(stop_cmd)
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- self.configure(env)
- start_cmd = format("service elasticsearch start")
- print 'Start the Slave'
- Execute(start_cmd)
-
- def status(self, env):
- import params
- env.set_params(params)
- status_cmd = format("service elasticsearch status")
- print 'Status of the Slave'
- Execute(status_cmd)
-
- def restart(self, env):
- import params
- env.set_params(params)
- self.configure(env)
- restart_cmd = format("service elasticsearch restart")
- print 'Restarting the Slave'
- Execute(restart_cmd)
-
-
-if __name__ == "__main__":
- Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
deleted file mode 100755
index c3e9169..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script import Script
-
-# server configurations
-config = Script.get_config()
-
-elastic_home = config['configurations']['elastic-sysconfig']['elastic_home']
-data_dir = config['configurations']['elastic-sysconfig']['data_dir']
-work_dir = config['configurations']['elastic-sysconfig']['work_dir']
-conf_dir = config['configurations']['elastic-sysconfig']['conf_dir']
-heap_size = config['configurations']['elastic-sysconfig']['heap_size']
-max_open_files = config['configurations']['elastic-sysconfig']['max_open_files']
-max_map_count = config['configurations']['elastic-sysconfig']['max_map_count']
-
-elastic_user = config['configurations']['elastic-env']['elastic_user']
-user_group = config['configurations']['elastic-env']['user_group']
-log_dir = config['configurations']['elastic-env']['elastic_log_dir']
-pid_dir = '/var/run/elasticsearch'
-pid_file = '/var/run/elasticsearch/elasticsearch.pid'
-hostname = config['hostname']
-java64_home = config['hostLevelParams']['java_home']
-elastic_env_sh_template = config['configurations']['elastic-env']['content']
-sysconfig_template = config['configurations']['elastic-sysconfig']['content']
-
-cluster_name = config['configurations']['elastic-site']['cluster_name']
-zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts']
-
-path_data = config['configurations']['elastic-site']['path_data']
-http_port = config['configurations']['elastic-site']['http_port']
-transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port']
-
-recover_after_time = config['configurations']['elastic-site']['recover_after_time']
-gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes']
-expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes']
-discovery_zen_ping_multicast_enabled = config['configurations']['elastic-site']['discovery_zen_ping_multicast_enabled']
-index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count']
-index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size']
-index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval']
-indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type']
-index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards']
-index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas']
-indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size']
-bootstrap_mlockall = config['configurations']['elastic-site']['bootstrap_mlockall']
-threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size']
-cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries']
-cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low']
-cluster_routing_allocation_disk_threshold_enabled = config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled']
-cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high']
-indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size']
-indices_cluster_send_refresh_mapping = config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping']
-threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size']
-
-discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout']
-discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval']
-discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout']
-discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries']
-
-network_host = config['configurations']['elastic-site']['network_host']
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
deleted file mode 100755
index ef9f6dd..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-
-
-def properties_inline_template(configurations):
- return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }}
-{% endfor %}''', configurations_dict=configurations)
-
-
-def properties_config(filename, configurations=None, conf_dir=None,
- mode=None, owner=None, group=None, brokerid=None):
- config_content = properties_inline_template(configurations)
- File(format("{conf_dir}/{filename}"), content=config_content, owner=owner,
- group=group, mode=mode)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
deleted file mode 100755
index 9615d83..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from __future__ import print_function
-
-import sys
-
-from resource_management.libraries.script import Script
-from resource_management.core.resources.system import Execute
-import subprocess
-
-
-class ServiceCheck(Script):
- def service_check(self, env):
- import params
- env.set_params(params)
-
- doc = '{"name": "Ambari Smoke test"}'
- index = "ambari_smoke_test"
-
- print("Running Elastic search service check", file=sys.stdout)
-
- # Make sure the service is actually up. We can live without everything allocated.
- # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can
- # also hit ES before things are green.
- host = "localhost:9200"
- Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host,
- logoutput=True,
- tries=6,
- try_sleep=20
- )
-
- # Put a document into a new index.
-
- Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True)
-
- # Retrieve the document. Use subprocess because we actually need the results here.
- cmd_retrieve = "curl -XGET '%s/%s/test/1'" % (host, index)
- proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
- response_retrieve = stdout
- print("Retrieval response is: %s" % response_retrieve)
- expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \
- % (index, doc)
-
- # Delete the index
- cmd_delete = "curl -XDELETE '%s/%s'" % (host, index)
- proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
- response_delete = stdout
- print("Delete index response is: %s" % response_retrieve)
- expected_delete = '{"acknowledged":true}'
-
- if (expected_retrieve == response_retrieve) and (expected_delete == response_delete):
- print("Smoke test able to communicate with Elasticsearch")
- else:
- print("Elasticsearch service unable to retrieve document.")
- sys.exit(1)
-
- exit(0)
-
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
deleted file mode 100755
index a134160..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-from resource_management.core.source import Template
-
-
-def slave():
- import params
-
- params.path_data = params.path_data.replace('"', '')
- data_path = params.path_data.replace(' ', '').split(',')
- data_path[:] = [x.replace('"', '') for x in data_path]
-
- directories = [params.log_dir, params.pid_dir, params.conf_dir]
- directories = directories + data_path
-
- Directory(directories,
- create_parents=True,
- mode=0755,
- owner=params.elastic_user,
- group=params.elastic_user,
- cd_access="a"
- )
-
- File("{}/elastic-env.sh".format(params.conf_dir),
- owner=params.elastic_user,
- content=InlineTemplate(params.elastic_env_sh_template)
- )
-
- configurations = params.config['configurations']['elastic-site']
-
- File("{}/elasticsearch.yml".format(params.conf_dir),
- content=Template(
- "elasticsearch.slave.yaml.j2",
- configurations=configurations),
- owner=params.elastic_user,
- group=params.elastic_user
- )
-
- print "Master sysconfig: /etc/sysconfig/elasticsearch"
- File(format("/etc/sysconfig/elasticsearch"),
- owner="root",
- group="root",
- content=InlineTemplate(params.sysconfig_template)
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
deleted file mode 100755
index 9cfb5cf..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script import Script
-
-config = Script.get_config()
-
-elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
-elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid")
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
deleted file mode 100755
index a9de018..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
+++ /dev/null
@@ -1,84 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-cluster:
- name: {{cluster_name}}
- routing:
- allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
- allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
- allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
- allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
-
-discovery:
- zen:
- ping:
- multicast:
- enabled: {{discovery_zen_ping_multicast_enabled}}
- unicast:
- hosts: "{{zen_discovery_ping_unicast_hosts}}"
-
-node:
- data: false
- master: true
- name: {{hostname}}
-path:
- data: {{path_data}}
-
-http.cors.enabled: true
-
-port: {{http_port}}
-
-transport:
- tcp:
- port: {{transport_tcp_port}}
-
-gateway:
- recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
- recover_after_time: {{recover_after_time}}
- expected_data_nodes: {{expected_data_nodes}}
-
-index:
- number_of_shards: {{index_number_of_shards}}
- merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}}
- translog.flush_threshold_size: {{index_translog_flush_threshold_size}}
- refresh_interval: {{index_refresh_interval}}
- number_of_replicas: {{index_number_of_replicas}}
-
-indices:
- memory:
- index_buffer_size: {{indices_memory_index_buffer_size}}
- store.throttle.type: {{indices_memory_index_store_throttle_type}}
- fielddata:
- cache.size: {{indices_fielddata_cache_size}}
- cluster:
- send_refresh_mapping: {{indices_cluster_send_refresh_mapping}}
-
-bootstrap.mlockall: {{bootstrap_mlockall}}
-
-threadpool:
- bulk:
- queue_size: {{threadpool_bulk_queue_size}}
- index:
- queue_size: {{threadpool_index_queue_size}}
-
-discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
-discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
-discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
-discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
-
-network.host: {{network_host}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
deleted file mode 100755
index e88fc5f..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
+++ /dev/null
@@ -1,84 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-cluster:
- name: {{cluster_name}}
- routing:
- allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
- allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
- allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
- allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
-
-discovery:
- zen:
- ping:
- multicast:
- enabled: {{discovery_zen_ping_multicast_enabled}}
- unicast:
- hosts: "{{zen_discovery_ping_unicast_hosts}}"
-
-node:
- data: true
- master: false
- name: {{hostname}}
-path:
- data: {{path_data}}
-
-http.cors.enabled: true
-
-port: {{http_port}}
-
-transport:
- tcp:
- port: {{transport_tcp_port}}
-
-gateway:
- recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
- recover_after_time: {{recover_after_time}}
- expected_data_nodes: {{expected_data_nodes}}
-
-index:
- number_of_shards: {{index_number_of_shards}}
- merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}}
- translog.flush_threshold_size: {{index_translog_flush_threshold_size}}
- refresh_interval: {{index_refresh_interval}}
- number_of_replicas: {{index_number_of_replicas}}
-
-indices:
- memory:
- index_buffer_size: {{indices_memory_index_buffer_size}}
- store.throttle.type: {{indices_memory_index_store_throttle_type}}
- fielddata:
- cache.size: {{indices_fielddata_cache_size}}
- cluster:
- send_refresh_mapping: {{indices_cluster_send_refresh_mapping}}
-
-bootstrap.mlockall: {{bootstrap_mlockall}}
-
-threadpool:
- bulk:
- queue_size: {{threadpool_bulk_queue_size}}
- index:
- queue_size: {{threadpool_index_queue_size}}
-
-discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
-discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
-discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
-discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
-
-network.host: {{network_host}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
deleted file mode 100755
index 130d018..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "_comment" : "Record format:",
- "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps" : {
- "_comment" : "dependencies for all cases",
- "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
deleted file mode 100755
index 3a1c8f7..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration supports_final="true">
- <property>
- <name>metron_home</name>
- <value>/usr/metron/0.2.0BETA</value>
- <description>Metron home directory</description>
- <display-name>Metron home</display-name>
- </property>
- <property>
- <name>metron_apps_hdfs_dir</name>
- <value>/apps/metron</value>
- <description>Metron apps HDFS dir</description>
- <display-name>Metron apps HDFS dir</display-name>
- </property>
- <property>
- <name>metron_user</name>
- <value>metron</value>
- <property-type>USER</property-type>
- <description>The user for Metron</description>
- <display-name>Metron User</display-name>
- </property>
- <property>
- <name>metron_group</name>
- <value>metron</value>
- <description>The group for Metron</description>
- </property>
- <property>
- <name>metron_indexing_topology</name>
- <value>indexing</value>
- <description>The Storm topology name for Indexing</description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
deleted file mode 100755
index d2dce5a..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
+++ /dev/null
@@ -1,105 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>INDEXING</name>
- <displayName>Indexing</displayName>
- <comment>Indexing</comment>
- <version>0.2.0BETA</version>
-
- <components>
- <component>
- <name>INDEXING_MASTER</name>
- <displayName>Indexing Master-Eligible Node</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>true</versionAdvertised>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>metron-common</name>
- </package>
- <package>
- <name>metron-indexing</name>
- </package>
- <package>
- <name>metron-elasticsearch</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>HDFS</service>
- <service>KAFKA</service>
- <service>STORM</service>
- <service>ZOOKEEPER</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>metron-indexing</config-type>
- </configuration-dependencies>
-
- <quickLinksConfigurations>
- <quickLinksConfiguration>
- <fileName>quicklinks.json</fileName>
- <default>true</default>
- </quickLinksConfiguration>
- </quickLinksConfigurations>
- </service>
- </services>
-</metainfo>