You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/09/12 14:48:51 UTC
incubator-metron git commit: METRON-385 Create Ambari Service
Definition for Indexing (justinleet via cestella) closes
apache/incubator-metron#222
Repository: incubator-metron
Updated Branches:
refs/heads/master 7eceac2aa -> 2e5538c4e
METRON-385 Create Ambari Service Definition for Indexing (justinleet via cestella) closes apache/incubator-metron#222
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/2e5538c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/2e5538c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/2e5538c4
Branch: refs/heads/master
Commit: 2e5538c4e7abd21aa73ecea4ef88c596e51896e0
Parents: 7eceac2
Author: justinleet <ju...@gmail.com>
Authored: Mon Sep 12 10:47:58 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Mon Sep 12 10:47:58 2016 -0400
----------------------------------------------------------------------
metron-deployment/packaging/ambari/.gitignore | 1 -
.../0.2.0BETA/configuration/metron-indexing.xml | 48 ++++++
.../INDEXING/0.2.0BETA/metainfo.xml | 105 +++++++++++++
.../0.2.0BETA/package/scripts/commands.py | 146 +++++++++++++++++++
.../package/scripts/indexing_master.py | 71 +++++++++
.../0.2.0BETA/package/scripts/params.py | 29 ++++
.../0.2.0BETA/package/scripts/params_linux.py | 70 +++++++++
.../0.2.0BETA/package/scripts/params_windows.py | 17 +++
.../0.2.0BETA/package/scripts/service_check.py | 37 +++++
.../0.2.0BETA/package/scripts/status_params.py | 19 +++
.../0.2.0BETA/quicklinks/quicklinks.json | 28 ++++
.../HDP/2.4/services/INDEXING/metainfo.xml | 26 ++++
.../services/INDEXING/role_command_order.json | 9 ++
.../src/main/config/elasticsearch.properties | 5 +-
14 files changed, 609 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore
index db8ded0..e708548 100644
--- a/metron-deployment/packaging/ambari/.gitignore
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -1,3 +1,2 @@
archive.zip
*.hash
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
new file mode 100755
index 0000000..3a1c8f7
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>metron_home</name>
+ <value>/usr/metron/0.2.0BETA</value>
+ <description>Metron home directory</description>
+ <display-name>Metron home</display-name>
+ </property>
+ <property>
+ <name>metron_apps_hdfs_dir</name>
+ <value>/apps/metron</value>
+ <description>Metron apps HDFS dir</description>
+ <display-name>Metron apps HDFS dir</display-name>
+ </property>
+ <property>
+ <name>metron_user</name>
+ <value>metron</value>
+ <property-type>USER</property-type>
+ <description>The user for Metron</description>
+ <display-name>Metron User</display-name>
+ </property>
+ <property>
+ <name>metron_group</name>
+ <value>metron</value>
+ <description>The group for Metron</description>
+ </property>
+ <property>
+ <name>metron_indexing_topology</name>
+ <value>indexing</value>
+ <description>The Storm topology name for Indexing</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
new file mode 100755
index 0000000..d2dce5a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
@@ -0,0 +1,105 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>INDEXING</name>
+ <displayName>Indexing</displayName>
+ <comment>Indexing</comment>
+ <version>0.2.0BETA</version>
+
+ <components>
+ <component>
+ <name>INDEXING_MASTER</name>
+ <displayName>Indexing Master-Eligible Node</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>metron-common</name>
+ </package>
+ <package>
+ <name>metron-indexing</name>
+ </package>
+ <package>
+ <name>metron-elasticsearch</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ <service>KAFKA</service>
+ <service>STORM</service>
+ <service>ZOOKEEPER</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>metron-indexing</config-type>
+ </configuration-dependencies>
+
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py
new file mode 100755
index 0000000..a9a86c0
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import subprocess
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+
+# Wrap major operations and functionality in this class
+class Commands:
+ __params = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__configured = os.path.isfile(self.__params.configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.0BETA]\n"
+ "name=Metron 0.2.0BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{}/kafka-topics.sh \
+ --zookeeper {} \
+ --create \
+ --topic {} \
+ --partitions {} \
+ --replication-factor {} \
+ --config retention.bytes={}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = 10
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating topics for indexing")
+
+ Logger.info("Creating topic'{}'".format(self.__params.metron_indexing_topology))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ self.__params.metron_indexing_topology,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_indexing_topology(self):
+ Logger.info("Starting Metron indexing topology: {}".format(self.__params.metron_indexing_topology))
+ start_cmd_template = """{}/bin/start_elasticsearch_topology.sh \
+ -s {} \
+ -z {}"""
+ Logger.info('Starting ' + self.__params.metron_indexing_topology)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__params.metron_indexing_topology, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting indexing topology')
+
+ def stop_indexing_topology(self):
+ Logger.info('Stopping ' + self.__params.metron_indexing_topology)
+ stop_cmd = 'storm kill ' + self.__params.metron_indexing_topology
+ Execute(stop_cmd)
+ Logger.info('Done stopping indexing topologies')
+
+ def restart_indexing_topology(self):
+ Logger.info('Restarting the indexing topologies')
+ self.stop_indexing_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active()
+ while topology_active and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(40)
+ topology_active = self.is_topology_active()
+ retries += 1
+
+ if not topology_active:
+ self.start_indexing_topology()
+ Logger.info('Done restarting the indexing topologies')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self):
+ cmd_retrieve = "storm list | grep 'indexing'"
+
+ proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+ Logger.info("Retrieval response is: %s" % stdout)
+ Logger.warning("Error response is: %s" % stderr)
+
+ fields = stdout.split()
+ if len(fields) < 2:
+ Logger.warning("Indexing topology is not running")
+ return False
+
+ # Get the second column, which is status. We already know first column is indexing)
+ status = stdout.split()[1]
+ running_status_set = ['ACTIVE', 'REBALANCING']
+ return status in running_status_set
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py
new file mode 100755
index 0000000..bfae19a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py
@@ -0,0 +1,71 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.libraries.script import Script
+
+from commands import Commands
+
+
+class Indexing(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+ commands = Commands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ commands = Commands(params)
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.set_configured()
+
+ commands.start_indexing_topology()
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ commands = Commands(params)
+ commands.stop_indexing_topology()
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ commands = Commands(status_params)
+
+ if not commands.is_topology_active():
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ import params
+ env.set_params(params)
+ commands = Commands(params)
+ commands.restart_indexing_topology()
+
+ def kafkabuild(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ commands = Commands(params)
+ commands.init_kafka_topics()
+
+
+if __name__ == "__main__":
+ Indexing().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py
new file mode 100755
index 0000000..6440005
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
+
+if OSCheck.is_windows_family():
+ pass
+else:
+ from params_linux import *
+
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py
new file mode 100755
index 0000000..86a0359
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.script import Script
+
+# Server configurations
+config = Script.get_config()
+
+hostname = config['hostname']
+metron_home = config['configurations']['metron-indexing']['metron_home']
+metron_indexing_topology = config['configurations']['metron-indexing']['metron_indexing_topology']
+yum_repo_type = 'local'
+metron_user = config['configurations']['metron-indexing']['metron_user']
+
+metron_config_path = metron_home + '/config'
+configured_flag_file = metron_config_path + '/metron_indexing_is_configured'
+
+# Hadoop params
+hadoop_home_dir = stack_select.get_hadoop_dir("home")
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+
+# Zookeeper
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+has_zk_host = not len(zk_hosts) == 0
+zookeeper_quorum = None
+if has_zk_host:
+ if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+ else:
+ zookeeper_clientPort = '2181'
+ zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
+ # last port config
+ zookeeper_quorum += ':' + zookeeper_clientPort
+
+# Kafka
+stack_root = Script.get_stack_root()
+kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+kafka_bin_dir = os.path.join(kafka_home, "bin")
+metron_indexing_topic_retention = config['configurations']['metron-indexing']['metron_indexing_topic_retention']
+
+kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
+has_kafka_host = not len(kafka_hosts) == 0
+kafka_brokers = None
+if has_kafka_host:
+ if 'port' in config['configurations']['kafka-broker']:
+ kafka_broker_port = config['configurations']['kafka-broker']['port']
+ else:
+ kafka_broker_port = '6667'
+ kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
+ kafka_brokers += ':' + kafka_broker_port
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py
new file mode 100755
index 0000000..b5828d2
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py
@@ -0,0 +1,17 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py
new file mode 100755
index 0000000..c02d695
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from __future__ import print_function
+
+from resource_management.libraries.script import Script
+
+from commands import Commands
+
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ commands = Commands(params)
+ if commands.is_topology_active():
+ exit(0)
+ else:
+ exit(1)
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py
new file mode 100755
index 0000000..30aed60
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+metron_indexing_topology = 'indexing'
+configured_flag_file = ""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json
new file mode 100755
index 0000000..ee1b225
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "storm_ui",
+ "label": "Storm UI",
+ "requires_user_name": "false",
+ "component_name": "STORM_UI_SERVER",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "ui.port",
+ "http_default_port": "8744",
+ "https_property": "ui.port",
+ "https_default_port": "8744",
+ "regex": "^(\\d+)$",
+ "site": "storm-site"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
new file mode 100755
index 0000000..e406780
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>INDEXING</name>
+ <version>0.2.0BETA</version>
+ <extends>common-services/INDEXING/0.2.0BETA</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
new file mode 100755
index 0000000..f58c59c
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
@@ -0,0 +1,9 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "INDEXING_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
+ "INDEXING_SERVICE_CHECK-SERVICE_CHECK" : ["INDEXING_MASTER-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
index a1516d6..c2c10af 100644
--- a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
+++ b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
@@ -57,10 +57,13 @@ org.apache.metron.metrics.TelemetryParserBolt.fails=true
bolt.hdfs.batch.size=5000
bolt.hdfs.field.delimiter=|
+bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
+bolt.hdfs.rotation.policy.units=DAYS
+bolt.hdfs.rotation.policy.count=1
+
bolt.hdfs.file.rotation.size.in.mb=5
bolt.hdfs.file.system.url=hdfs://iot01.cloud.hortonworks.com:8020
bolt.hdfs.wip.file.path=/paloalto/wip
bolt.hdfs.finished.file.path=/paloalto/rotated
bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
index.hdfs.output=/tmp/metron/enriched
-