You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/04/05 15:08:01 UTC
[1/7] metron git commit: METRON-1462: Separate ES and Kibana from
Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Repository: metron
Updated Branches:
refs/heads/master 0d847cf5f -> 0ab39a32b
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
new file mode 100755
index 0000000..f0903ac
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
@@ -0,0 +1,95 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from elasticsearch import Elasticsearch
+from elasticsearch.helpers import bulk
+import cPickle as pickle
+import argparse, sys, os.path
+import errno
+import os
+
+
+class DashboardIndex(object):
+
+ def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs):
+ """
+ :arg host: hostname of the node (default: localhost)
+ :arg port: port to use (integer, default: 9200)
+ :arg url_prefix: optional url prefix for elasticsearch
+ :arg timeout: default timeout in seconds (float, default: 10)
+ """
+ self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}])
+
+ def get(self):
+ """
+ Get .kibana index from Elasticsearch
+ """
+ dotkibana = self.es.search(index='.kibana', size = 100)
+ return dotkibana['hits']['hits']
+
+ def load(self,filespec):
+ """
+ Save Index data on local filesystem
+ :args filespec: path/filename for saved file
+ """
+ data=[]
+ with open(filespec,'rb') as fp:
+ data = pickle.load(fp)
+ return data
+
+ def save(self,filename,data):
+ """
+ Save Index data on local filesystem
+ :args filespec: path/filename for saved file
+ """
+ with open(filename,'wb') as fp:
+ pickle.dump(data,fp)
+
+ def put(self,data):
+ """
+ Bulk write data to Elasticsearch
+ :args data: data to be written (note: index name is specified in data)
+ """
+ bulk(self.es,data)
+
+ def main(self,args):
+
+ if args.save:
+ print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec))
+ self.save(filename=args.filespec,data=di.get())
+ else:
+ """
+ Loads Kibana Dashboard definition from disk and replaces .kibana on index
+ :args filespec: path/filename for saved file
+ """
+ if not os.path.isfile(args.filespec):
+ raise IOError(
+ errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
+ self.es.indices.delete(index='.kibana', ignore=[400, 404])
+ self.put(data=di.load(filespec=args.filespec))
+
+if __name__ == '__main__':
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("hostname", help="ES Hostname or IP", type=str)
+ parser.add_argument("port", help="ES Port", type=int)
+ parser.add_argument("filespec", help="file to be pushed from or saved to", type=str)
+ parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true")
+ args = parser.parse_args()
+ di = DashboardIndex(host=args.hostname,port=args.port)
+ di.main(args)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
new file mode 100644
index 0000000..6f38ed5
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
@@ -0,0 +1,233 @@
+{
+ "template" : ".kibana",
+ "mappings" : {
+ "search" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "columns" : {
+ "type" : "keyword"
+ },
+ "description" : {
+ "type" : "text"
+ },
+ "hits" : {
+ "type" : "integer"
+ },
+ "kibanaSavedObjectMeta" : {
+ "properties" : {
+ "searchSourceJSON" : {
+ "type" : "text"
+ }
+ }
+ },
+ "sort" : {
+ "type" : "keyword"
+ },
+ "title" : {
+ "type" : "text"
+ },
+ "version" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "url" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "accessCount" : {
+ "type" : "long"
+ },
+ "accessDate" : {
+ "type" : "date"
+ },
+ "createDate" : {
+ "type" : "date"
+ },
+ "url" : {
+ "type" : "text",
+ "fields" : {
+ "keyword" : {
+ "type" : "keyword",
+ "ignore_above" : 2048
+ }
+ }
+ }
+ }
+ },
+ "dashboard" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "description" : {
+ "type" : "text"
+ },
+ "hits" : {
+ "type" : "integer"
+ },
+ "kibanaSavedObjectMeta" : {
+ "properties" : {
+ "searchSourceJSON" : {
+ "type" : "text"
+ }
+ }
+ },
+ "optionsJSON" : {
+ "type" : "text"
+ },
+ "panelsJSON" : {
+ "type" : "text"
+ },
+ "refreshInterval" : {
+ "properties" : {
+ "display" : {
+ "type" : "keyword"
+ },
+ "pause" : {
+ "type" : "boolean"
+ },
+ "section" : {
+ "type" : "integer"
+ },
+ "value" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "timeFrom" : {
+ "type" : "keyword"
+ },
+ "timeRestore" : {
+ "type" : "boolean"
+ },
+ "timeTo" : {
+ "type" : "keyword"
+ },
+ "title" : {
+ "type" : "text"
+ },
+ "uiStateJSON" : {
+ "type" : "text"
+ },
+ "version" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "index-pattern" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "fieldFormatMap" : {
+ "type" : "text"
+ },
+ "fields" : {
+ "type" : "text"
+ },
+ "intervalName" : {
+ "type" : "keyword"
+ },
+ "notExpandable" : {
+ "type" : "boolean"
+ },
+ "sourceFilters" : {
+ "type" : "text"
+ },
+ "timeFieldName" : {
+ "type" : "keyword"
+ },
+ "title" : {
+ "type" : "text"
+ }
+ }
+ },
+ "timelion-sheet" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "description" : {
+ "type" : "text"
+ },
+ "hits" : {
+ "type" : "integer"
+ },
+ "kibanaSavedObjectMeta" : {
+ "properties" : {
+ "searchSourceJSON" : {
+ "type" : "text"
+ }
+ }
+ },
+ "timelion_chart_height" : {
+ "type" : "integer"
+ },
+ "timelion_columns" : {
+ "type" : "integer"
+ },
+ "timelion_interval" : {
+ "type" : "keyword"
+ },
+ "timelion_other_interval" : {
+ "type" : "keyword"
+ },
+ "timelion_rows" : {
+ "type" : "integer"
+ },
+ "timelion_sheet" : {
+ "type" : "text"
+ },
+ "title" : {
+ "type" : "text"
+ },
+ "version" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "visualization" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "description" : {
+ "type" : "text"
+ },
+ "kibanaSavedObjectMeta" : {
+ "properties" : {
+ "searchSourceJSON" : {
+ "type" : "text"
+ }
+ }
+ },
+ "savedSearchId" : {
+ "type" : "keyword"
+ },
+ "title" : {
+ "type" : "text"
+ },
+ "uiStateJSON" : {
+ "type" : "text"
+ },
+ "version" : {
+ "type" : "integer"
+ },
+ "visState" : {
+ "type" : "text"
+ }
+ }
+ },
+ "server" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "uuid" : {
+ "type" : "keyword"
+ }
+ }
+ },
+ "_default_" : {
+ "dynamic" : "strict"
+ },
+ "config" : {
+ "dynamic" : "true",
+ "properties" : {
+ "buildNum" : {
+ "type" : "keyword"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index ce8c074..1cd6f4c 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -14,8 +14,12 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+import errno
import os
import requests
+
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
from resource_management.core.exceptions import ComponentIsNotRunning
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute
@@ -149,6 +153,34 @@ class Indexing(Script):
cmd.format(params.es_http_url, template_name),
logoutput=True)
+ @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+ def kibana_dashboard_install(self, env):
+ from params import params
+ env.set_params(params)
+
+ Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url))
+
+ kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template')
+ if not os.path.isfile(kibanaTemplate):
+ raise IOError(
+ errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate)
+
+ Logger.info("Loading .kibana index template from %s" % kibanaTemplate)
+ template_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_http_url}/_template/.kibana -d @%s' % kibanaTemplate)
+ Execute(template_cmd, logoutput=True)
+
+ kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json')
+ if not os.path.isfile(kibanaDashboardLoad):
+ raise IOError(
+ errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad)
+
+ Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad)
+
+ kibana_cmd = ambari_format(
+ 'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_http_url}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad)
+ Execute(kibana_cmd, logoutput=True)
+
def zeppelin_notebook_import(self, env):
from params import params
env.set_params(params)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml
index fe76ade..f59324c 100644
--- a/metron-deployment/pom.xml
+++ b/metron-deployment/pom.xml
@@ -49,6 +49,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<modules>
+ <module>packaging/ambari/elasticsearch-mpack</module>
<module>packaging/ambari/metron-mpack</module>
</modules>
</profile>
[7/7] metron git commit: METRON-1462: Separate ES and Kibana from
Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Posted by mm...@apache.org.
METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0ab39a32
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0ab39a32
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0ab39a32
Branch: refs/heads/master
Commit: 0ab39a32b61104812a6a9e69e999e7053e3e7e90
Parents: 0d847cf
Author: mmiklavc <mi...@gmail.com>
Authored: Thu Apr 5 09:07:48 2018 -0600
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Thu Apr 5 09:07:48 2018 -0600
----------------------------------------------------------------------
.../roles/ambari_master/defaults/main.yml | 2 +
.../ambari_master/tasks/elasticsearch_mpack.yml | 26 ++
.../ansible/roles/ambari_master/tasks/main.yml | 3 +-
.../roles/ambari_master/tasks/metron_mpack.yml | 26 ++
.../ansible/roles/ambari_master/tasks/mpack.yml | 26 --
.../roles/load_web_templates/tasks/main.yml | 2 +-
.../manual-install/Manual_Install_CentOS6.md | 4 +-
metron-deployment/packaging/ambari/README.md | 25 +-
.../ambari/elasticsearch-mpack/README.md | 62 +++++
.../ambari/elasticsearch-mpack/pom.xml | 95 +++++++
.../src/main/assemblies/elasticsearch-mpack.xml | 43 +++
.../ELASTICSEARCH/5.6.2/metainfo.xml | 29 ++
.../ELASTICSEARCH/5.6.2/repos/repoinfo.xml | 45 ++++
.../addon-services/KIBANA/5.6.2/metainfo.xml | 30 +++
.../KIBANA/5.6.2/quicklinks/quicklinks.json | 27 ++
.../KIBANA/5.6.2/repos/repoinfo.xml | 60 +++++
.../5.6.2/configuration/elastic-env.xml | 86 ++++++
.../5.6.2/configuration/elastic-jvm-options.xml | 144 ++++++++++
.../5.6.2/configuration/elastic-site.xml | 198 ++++++++++++++
.../5.6.2/configuration/elastic-sysconfig.xml | 97 +++++++
.../5.6.2/configuration/elastic-systemd.xml | 30 +++
.../ELASTICSEARCH/5.6.2/metainfo.xml | 97 +++++++
.../5.6.2/package/scripts/elastic_commands.py | 266 +++++++++++++++++++
.../5.6.2/package/scripts/elastic_master.py | 72 +++++
.../5.6.2/package/scripts/elastic_slave.py | 71 +++++
.../5.6.2/package/scripts/params.py | 108 ++++++++
.../5.6.2/package/scripts/properties_config.py | 34 +++
.../5.6.2/package/scripts/service_check.py | 114 ++++++++
.../5.6.2/package/scripts/status_params.py | 27 ++
.../templates/elasticsearch.master.yaml.j2 | 77 ++++++
.../templates/elasticsearch.slave.yaml.j2 | 78 ++++++
.../templates/elasticsearch_limits.conf.j2 | 20 ++
.../5.6.2/quicklinks/quicklinks.json | 43 +++
.../ELASTICSEARCH/5.6.2/role_command_order.json | 8 +
.../KIBANA/5.6.2/configuration/kibana-env.xml | 72 +++++
.../KIBANA/5.6.2/configuration/kibana-site.xml | 113 ++++++++
.../common-services/KIBANA/5.6.2/metainfo.xml | 84 ++++++
.../KIBANA/5.6.2/package/scripts/common.py | 56 ++++
.../5.6.2/package/scripts/kibana_master.py | 81 ++++++
.../KIBANA/5.6.2/package/scripts/params.py | 50 ++++
.../KIBANA/5.6.2/quicklinks/quicklinks.json | 28 ++
.../src/main/resources/mpack.json | 76 ++++++
.../packaging/ambari/metron-mpack/README.md | 20 +-
.../src/main/assemblies/metron-mpack.xml | 14 -
.../ELASTICSEARCH/5.6.2/metainfo.xml | 29 --
.../ELASTICSEARCH/5.6.2/repos/repoinfo.xml | 45 ----
.../addon-services/KIBANA/5.6.2/metainfo.xml | 30 ---
.../KIBANA/5.6.2/quicklinks/quicklinks.json | 27 --
.../KIBANA/5.6.2/repos/repoinfo.xml | 60 -----
.../5.6.2/configuration/elastic-env.xml | 86 ------
.../5.6.2/configuration/elastic-jvm-options.xml | 144 ----------
.../5.6.2/configuration/elastic-site.xml | 198 --------------
.../5.6.2/configuration/elastic-sysconfig.xml | 97 -------
.../5.6.2/configuration/elastic-systemd.xml | 30 ---
.../ELASTICSEARCH/5.6.2/metainfo.xml | 97 -------
.../5.6.2/package/scripts/elastic_commands.py | 266 -------------------
.../5.6.2/package/scripts/elastic_master.py | 72 -----
.../5.6.2/package/scripts/elastic_slave.py | 71 -----
.../5.6.2/package/scripts/params.py | 108 --------
.../5.6.2/package/scripts/properties_config.py | 34 ---
.../5.6.2/package/scripts/service_check.py | 114 --------
.../5.6.2/package/scripts/status_params.py | 27 --
.../templates/elasticsearch.master.yaml.j2 | 77 ------
.../templates/elasticsearch.slave.yaml.j2 | 78 ------
.../templates/elasticsearch_limits.conf.j2 | 20 --
.../5.6.2/quicklinks/quicklinks.json | 43 ---
.../ELASTICSEARCH/5.6.2/role_command_order.json | 8 -
.../KIBANA/5.6.2/configuration/kibana-env.xml | 72 -----
.../KIBANA/5.6.2/configuration/kibana-site.xml | 113 --------
.../common-services/KIBANA/5.6.2/metainfo.xml | 94 -------
.../KIBANA/5.6.2/package/scripts/common.py | 56 ----
.../5.6.2/package/scripts/dashboard/__init__.py | 16 --
.../scripts/dashboard/dashboard-bulkload.json | 88 ------
.../package/scripts/dashboard/dashboardindex.py | 95 -------
.../package/scripts/dashboard/kibana.template | 233 ----------------
.../5.6.2/package/scripts/kibana_master.py | 119 ---------
.../KIBANA/5.6.2/package/scripts/params.py | 50 ----
.../KIBANA/5.6.2/quicklinks/quicklinks.json | 28 --
.../common-services/METRON/CURRENT/metainfo.xml | 9 +
.../package/scripts/dashboard/__init__.py | 16 ++
.../scripts/dashboard/dashboard-bulkload.json | 88 ++++++
.../package/scripts/dashboard/dashboardindex.py | 95 +++++++
.../package/scripts/dashboard/kibana.template | 233 ++++++++++++++++
.../CURRENT/package/scripts/indexing_master.py | 32 +++
metron-deployment/pom.xml | 1 +
85 files changed, 3082 insertions(+), 2786 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
index 0719b7f..52d220a 100644
--- a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
@@ -19,3 +19,5 @@ ambari_server_mem: 2048
ambari_mpack_version: 0.4.3.0
metron_mpack_name: metron_mpack-{{ ambari_mpack_version }}.tar.gz
metron_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/metron-mpack/target/{{ metron_mpack_name }}"
+elasticsearch_mpack_name: elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz
+elasticsearch_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/elasticsearch-mpack/target/{{ elasticsearch_mpack_name }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
new file mode 100644
index 0000000..7ce1a13
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Copy Elasticsearch MPack to Ambari Host
+ copy:
+ src: "{{ elasticsearch_mpack_path }}"
+ dest: /tmp
+
+- name: Install Elasticsearch MPack on Ambari Host
+ shell: ambari-server install-mpack --mpack=/tmp/elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz
+ args:
+ creates: /var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-{{ ambari_mpack_version }}/addon-services
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/ambari_master/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/main.yml b/metron-deployment/ansible/roles/ambari_master/tasks/main.yml
index 77a2e74..b6c351d 100644
--- a/metron-deployment/ansible/roles/ambari_master/tasks/main.yml
+++ b/metron-deployment/ansible/roles/ambari_master/tasks/main.yml
@@ -16,7 +16,8 @@
#
---
- include: ambari.yml
-- include: mpack.yml
+- include: metron_mpack.yml
+- include: elasticsearch_mpack.yml
- name: start ambari server
service:
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/ambari_master/tasks/metron_mpack.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/metron_mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/metron_mpack.yml
new file mode 100644
index 0000000..a44ea17
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_master/tasks/metron_mpack.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Copy Metron MPack to Ambari Host
+ copy:
+ src: "{{ metron_mpack_path }}"
+ dest: /tmp
+
+- name: Install Metron MPack on Ambari Host
+ shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-{{ ambari_mpack_version }}.tar.gz
+ args:
+ creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-{{ ambari_mpack_version }}/addon-services
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml
deleted file mode 100644
index 16289bb..0000000
--- a/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Copy MPack to Ambari Host
- copy:
- src: "{{ metron_mpack_path }}"
- dest: /tmp
-
-- name: Install MPack on Ambari Host
- shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-{{ ambari_mpack_version }}.tar.gz
- args:
- creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-{{ ambari_mpack_version }}/addon-services
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml b/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml
index 3a91960..55927c3 100644
--- a/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml
+++ b/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml
@@ -17,7 +17,7 @@
---
- name: Load Kibana Dashboard
command: >
- curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "{{ kibana_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests
+ curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "KIBANA_DASHBOARD_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "{{ metron_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests
args:
warn: off
register: result
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md
----------------------------------------------------------------------
diff --git a/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md b/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md
index e1305b6..1631fac 100644
--- a/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md
+++ b/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md
@@ -878,11 +878,11 @@ curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d
- Load Kibana Dashboard with:
```
-curl -s -w "%{http_code}" -u <USERNAME>:<PASSWORD> -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "<HOSTNAME>"}]}' http://<AMBARI HOST>:8080/api/v1/clusters/<CLUSTERNAME>/requests
+curl -s -w "%{http_code}" -u <USERNAME>:<PASSWORD> -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "KIBANA_DASHBOARD_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "<HOSTNAME>"}]}' http://<AMBARI HOST>:8080/api/v1/clusters/<CLUSTERNAME>/requests
```
For example:
```
-curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "metron"}]}' http://192.168.10.10:8080/api/v1/clusters/metron/requests
+curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "KIBANA_DASHBOARD_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "metron"}]}' http://192.168.10.10:8080/api/v1/clusters/metron/requests
```
- If you installed Metron on a single node, you might have to increase the number of Storm supervisor slots from the default 2 to 5 or more. This can be done by editing the "supervisor.slots.ports" under Storm in the Ambari UI.
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/README.md b/metron-deployment/packaging/ambari/README.md
index 82e1537..52c7570 100644
--- a/metron-deployment/packaging/ambari/README.md
+++ b/metron-deployment/packaging/ambari/README.md
@@ -30,13 +30,19 @@ limitations under the License.
* [Upgrading MPack Services](#upgrading-mpack-services)
## Overview
+
Typically, Ambari Management Pack development will be done in the Vagrant environments. These instructions are specific to Vagrant, but can be adapted for other environemnts (e.g. make sure to be on the correct nodes for server vs agent files)
-There is an `mpack.json` file which describes what services the mpack will contains, versions, etc.
+There are two MPacks:
+
+* Metron - contains artifacts for deploying the Metron service
+* Elasticsearch - contains artifacts for installing Elasticsearch and Kibana services
-Alongside this are two directories, `addon-services` and `common-services`.
+There is an `mpack.json` file for each which describes what services the mpack will contain, versions, etc.
-The layout of `/common-services/METRON.CURRENT` is
+Alongside this are two directories, `addon-services` and `common-services`. Below the Metron MPack is described, but this also applies similarly to the Elasticsearch MPack.
+
+The layout of `/common-services/METRON/CURRENT` is
* `/configuration`
* This contains a set of `*-env.xml` files, relevent to particular components or the service as a whole. These are where properties are defined.
* `/package`
@@ -59,11 +65,11 @@ The layout of `/common-services/METRON.CURRENT` is
* `service_advisor.py`
* Handles component layout and validation, along with handling some configurations for other services or that needs configs from other services.
-The layout of `/addon-services/METRON.CURRENT` is
+The layout of `/addon-services/METRON/CURRENT` is
* `/repos`
* Contains `repoinfo.xml` that defines repositories to install packages from
* `metainfo.xml`
- * Limited info version of `/common-services/METRON.CURRENT/metainfo.xml`
+ * Limited info version of `/common-services/METRON/CURRENT/metainfo.xml`
* `role_command_order.json`
* Defines the order of service startup and other actions relative to each other.
@@ -361,6 +367,7 @@ Ambari stores the Python files from the service in a couple places. We'll want t
Specifically, the server files live in
```
/var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-0.4.0.0/common-services
+/var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-0.4.0.0/common-services
/var/lib/ambari-agent/cache/common-services
```
@@ -409,6 +416,8 @@ After we've modified files in Ambari and the mpack is working, it is a good idea
```
1. Install the mpack through Ambari as you normally would
+1. The same steps can be followed for Elasticsearch and Kibana by similary deploying the ES MPack located in elasticsearch-mpack/target.
+
## Configuration involving dependency services
Metron can define expectations on other services, e.g. Storm's `topology.classpath` should be `/etc/hbase/conf:/etc/hadoop/conf`.
This happens in `METRON.CURRENT/service_advisor.py`.
@@ -468,10 +477,12 @@ This is checked in the indexing master
## Upgrading MPack Services
-Apache Metron currently provides three services as part of its MPack
+Apache Metron currently provides one service as part of its Metron MPack
+* Metron
+
+Apache Metron currently provides two services as part of its Elasticsearch MPack
* Elasticsearch
* Kibana
-* Metron
There is currently no mechanism provided for multi-version or backwards compatibility. If you upgrade a service, e.g. Elasticsearch 2.x to 5.x, that is the only version that will be
supported by Ambari via MPack.
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/README.md b/metron-deployment/packaging/ambari/elasticsearch-mpack/README.md
new file mode 100644
index 0000000..e9a20cc
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/README.md
@@ -0,0 +1,62 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+This provides a Management Pack (MPack) extension for [Apache Ambari](https://ambari.apache.org/) that simplifies the provisioning, management and monitoring of Elasticsearch and Kibana on clusters of any size.
+
+This allows you to easily install Elasticsearch and Kibana using a simple, guided process. This also allows you to monitor cluster health.
+
+### Prerequisites
+
+* Ambari 2.4.2+
+
+### Quick Start
+
+1. Build the Elasticsearch MPack. Execute the following command from the project's root directory.
+ ```
+ mvn clean package -Pmpack -DskipTests
+ ```
+
+1. This results in the Mpack being produced at the following location.
+ ```
+ metron-deployment/packaging/ambari/elasticsearch-mpack/target/elasticsearch_mpack-x.y.z.0.tar.gz
+ ```
+
+1. Copy the tarball to the host where Ambari Server is installed.
+
+1. Ensure that Ambari Server is stopped.
+
+1. Install the MPack.
+ ```
+ ambari-server install-mpack --mpack=elasticsearch_mpack-x.y.z.0.tar.gz --verbose
+ ```
+
+1. Elasticsearch and Kibana will now be available as an installable service within Ambari.
+
+### Installation Notes
+
+The MPack will make all Elasticsearch services available in Ambari in the same manner as any other services in a stack. These can be installed using Ambari's user interface using "Add Services" or during an initial cluster install.
+
+#### Kerberization
+
+Elasticsearch does not provide free native Kerberos support.
+
+#### Limitations
+
+There are a few limitations that should be addressed to improve the Elasticsearch MPack installation.
+
+* The MPack does not support upgrades.
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml
new file mode 100644
index 0000000..931ce63
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml
@@ -0,0 +1,95 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.metron.packaging.mpacks</groupId>
+ <artifactId>elasticsearch_mpack</artifactId>
+ <version>0.4.3.0</version>
+ <name>Elasticsearch Ambari Management Pack</name>
+
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-deployment</artifactId>
+ <version>0.4.3</version>
+ <relativePath>../../..</relativePath>
+ </parent>
+
+ <dependencies>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>mpack.json</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${global_jar_version}</version>
+ <executions>
+ <execution>
+ <id>default-jar</id>
+ <!-- put the default-jar in the none phase to skip it from being created -->
+ <phase>none</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/main/assemblies/elasticsearch-mpack.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/assemblies/elasticsearch-mpack.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/assemblies/elasticsearch-mpack.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/assemblies/elasticsearch-mpack.xml
new file mode 100644
index 0000000..238b028
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/assemblies/elasticsearch-mpack.xml
@@ -0,0 +1,43 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+ <id>archive</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>src/main/resources/common-services</directory>
+ <outputDirectory>common-services</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources/addon-services</directory>
+ <outputDirectory>addon-services</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <files>
+ <file>
+ <source>target/mpack.json</source>
+ </file>
+ </files>
+ <dependencySets>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml
new file mode 100755
index 0000000..accf7da
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ELASTICSEARCH</name>
+ <version>5.6.2</version>
+ <extends>common-services/ELASTICSEARCH/5.6.2</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
new file mode 100644
index 0000000..ba21fb1
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<reposinfo>
+ <os family="redhat6">
+ <repo>
+ <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
+ <repoid>elasticsearch-5.x</repoid>
+ <reponame>ELASTICSEARCH</reponame>
+ </repo>
+ </os>
+ <os family="redhat7">
+ <repo>
+ <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
+ <repoid>elasticsearch-5.x</repoid>
+ <reponame>ELASTICSEARCH</reponame>
+ </repo>
+ </os>
+ <os family="ubuntu14">
+ <!--
+ see details about Ambari fixes for Ubuntu introduced in Ambari 2.6+
+ https://github.com/apache/ambari/commit/f8b29df9685b443d4a5c06c6e1725e4428c95b49#diff-6f26c26ed59462200d018c5e1e71e773
+ https://issues.apache.org/jira/browse/AMBARI-21856
+ -->
+ <repo>
+ <baseurl>https://artifacts.elastic.co/packages/5.x/apt</baseurl>
+ <repoid>elasticsearch-5.x</repoid>
+ <reponame>ELASTICSEARCH</reponame>
+ <distribution>stable</distribution>
+ </repo>
+ </os>
+</reposinfo>
+
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml
new file mode 100755
index 0000000..8a4fba2
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>KIBANA</name>
+ <version>5.6.2</version>
+ <extends>common-services/KIBANA/5.6.2</extends>
+ </service>
+ </services>
+</metainfo>
+
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json
new file mode 100755
index 0000000..622a512
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json
@@ -0,0 +1,27 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "kibana_master_ui",
+ "label": "Metron Dashboard",
+ "requires_user_name": "false",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "kibana_server_port",
+ "http_default_port": "5601",
+ "https_property": "kibana_server_port",
+ "https_default_port": "5601",
+ "regex": "^(\\d+)$",
+ "site": "kibana-env"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
new file mode 100644
index 0000000..2755818
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<reposinfo>
+ <os family="redhat6">
+ <repo>
+ <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
+ <repoid>kibana-5.x</repoid>
+ <reponame>KIBANA</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://packages.elastic.co/curator/5/centos/6</baseurl>
+ <repoid>ES-Curator-5.x</repoid>
+ <reponame>CURATOR</reponame>
+ </repo>
+ </os>
+ <os family="redhat7">
+ <repo>
+ <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
+ <repoid>kibana-5.x</repoid>
+ <reponame>KIBANA</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://packages.elastic.co/curator/5/centos/7</baseurl>
+ <repoid>ES-Curator-5.x</repoid>
+ <reponame>CURATOR</reponame>
+ </repo>
+ </os>
+ <os family="ubuntu14">
+ <!--
+ see details about Ambari fixes for Ubuntu introduced in Ambari 2.6+
+ https://github.com/apache/ambari/commit/f8b29df9685b443d4a5c06c6e1725e4428c95b49#diff-6f26c26ed59462200d018c5e1e71e773
+ https://issues.apache.org/jira/browse/AMBARI-21856
+ -->
+ <repo>
+ <baseurl>https://artifacts.elastic.co/packages/5.x/apt</baseurl>
+ <repoid>kibana-5.x</repoid>
+ <reponame>KIBANA</reponame>
+ <distribution>stable</distribution>
+ </repo>
+ <repo>
+ <baseurl>https://packages.elastic.co/curator/5/debian</baseurl>
+ <repoid>ES-Curator-5.x</repoid>
+ <reponame>CURATOR</reponame>
+ <distribution>stable</distribution>
+ </repo>
+ </os>
+</reposinfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml
new file mode 100755
index 0000000..9e4f8ad
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>elastic_user</name>
+ <value>elasticsearch</value>
+ <property-type>USER</property-type>
+ <description>Service user for Elasticsearch</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ </property>
+ <property>
+ <name>elastic_group</name>
+ <value>elasticsearch</value>
+ <property-type>GROUP</property-type>
+ <description>Service group for Elasticsearch</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ </property>
+ <property>
+ <name>elastic_log_dir</name>
+ <value>/var/log/elasticsearch</value>
+ <description>Log directory for elastic</description>
+ </property>
+ <property>
+ <name>elastic_pid_dir</name>
+ <value>/var/run/elasticsearch</value>
+ <description>The directory for pid files</description>
+ </property>
+ <!-- elasticsearch-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for elastic-env.sh file</description>
+ <value>
+#!/bin/bash
+
+# Set ELASTICSEARCH specific environment variables here.
+
+# The java implementation to use.
+export JAVA_HOME={{java64_home}}
+export PATH=$PATH:$JAVA_HOME/bin
+ </value>
+ </property>
+ <property>
+ <name>elastic_user_nofile_limit</name>
+ <value>65536</value>
+ <description>Max open file limit for Elasticsearch user.</description>
+ </property>
+ <property>
+ <name>elastic_user_nproc_limit</name>
+ <value>2048</value>
+ <description>Max number of processes for Elasticsearch user.</description>
+ </property>
+ <property>
+ <name>elastic_user_memlock_soft_limit</name>
+ <value>unlimited</value>
+ <description>Max locked-in memory address space (soft memlock limit).</description>
+ </property>
+ <property>
+ <name>elastic_user_memlock_hard_limit</name>
+ <value>unlimited</value>
+ <description>Max locked-in memory address space (hard memlock limit).</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
new file mode 100644
index 0000000..5c6aaca
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>heap_size</name>
+ <value>512m</value>
+ <description>JVM heap size</description>
+ </property>
+ <property>
+ <name>content</name>
+ <description>The jinja template for the Elasticsearch JVM options file.</description>
+ <value>
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms{{heap_size}}
+-Xmx{{heap_size}}
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## basic
+
+# force the server VM (remove on 32-bit client JVMs)
+-server
+
+# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
+-Xss1m
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+# use old-style file permissions on JDK9
+-Djdk.io.permissionsUseCanonicalPath=true
+
+# flags to configure Netty
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
+
+# log4j 2
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+-Dlog4j.skipJansi=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${heap.dump.path}
+
+## GC logging
+
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${loggc}
+
+# By default, the GC log file will not rotate.
+# By uncommenting the lines below, the GC log file
+# will be rotated every 128MB at most 32 times.
+#-XX:+UseGCLogFileRotation
+#-XX:NumberOfGCLogFiles=32
+#-XX:GCLogFileSize=128M
+
+# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
+# If documents were already indexed with unquoted fields in a previous version
+# of Elasticsearch, some operations may throw errors.
+#
+# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
+# only for migration purposes.
+#-Delasticsearch.json.allow_unquoted_field_names=true
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
new file mode 100755
index 0000000..34df1e4
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
@@ -0,0 +1,198 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Elastic search Configurations -->
+
+<configuration supports_final="true">
+ <!-- Configurations -->
+ <property>
+ <name>cluster_name</name>
+ <value>metron</value>
+ <description>Elasticsearch Cluster Name identifies your Elasticsearch subsystem</description>
+ </property>
+ <property>
+ <name>masters_also_are_datanodes</name>
+ <value>"false"</value>
+ <description>ES Masters and Slaves cannot be installed on the same nodes. Set this to "true" if you want the ES master nodes to serve as combined master/datanodes. Note: surround value in quotes.</description>
+ <value-attributes>
+ <type>string</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>zen_discovery_ping_unicast_hosts</name>
+ <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.-->
+ <!--Also need to document whether should list masters only, or all ES nodes. I think this one is all nodes, but previous inline comment said Masters.-->
+ <value></value>
+ <description>Unicast discovery list of hosts to act as gossip routers, comma-separated list with square brackets: [ eshost1, eshost2 ]</description>
+ </property>
+ <property>
+ <name>index_number_of_shards</name>
+ <value>4</value>
+ <description>Set the number of shards (splits) of an index. Changes are not effective after index creation. Usually set to 1 for single-node install.</description>
+ </property>
+ <property>
+ <name>index_number_of_replicas</name>
+ <value>2</value>
+ <description>Set the number of replicas (copies in addition to the first) of an index. Usually set to 0 for single-node install.</description>
+ </property>
+ <property>
+ <name>path_data</name>
+ <value>"/opt/lmm/es_data"</value>
+ <description>Comma-separated list of directories where to store index data allocated for each node: "/mnt/first","/mnt/second". Number of paths should relate to number of shards, and preferably should be on separate physical volumes.</description>
+ </property>
+ <property>
+ <name>http_cors_enabled</name>
+ <value>"false"</value>
+ <description>Enable or disable cross-origin resource sharing, i.e. whether a browser on another origin can do requests to Elasticsearch. Defaults to false.</description>
+ <value-attributes>
+ <type>string</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>http_port</name>
+ <value>9200-9300</value>
+ <description>Set a custom port to listen for HTTP traffic</description>
+ </property>
+ <property>
+ <name>transport_tcp_port</name>
+ <value>9300-9400</value>
+ <description>Set a custom port for the node to node communication</description>
+ </property>
+ <!-- Multi-node Discovery -->
+ <property>
+ <name>discovery_zen_ping_timeout</name>
+ <value>3s</value>
+ <description>Wait for ping responses for master discovery</description>
+ </property>
+ <property>
+ <name>discovery_zen_fd_ping_interval</name>
+ <value>15s</value>
+ <description>Wait for ping for cluster discovery</description>
+ </property>
+ <property>
+ <name>discovery_zen_fd_ping_timeout</name>
+ <value>60s</value>
+ <description>Wait for ping for cluster discovery</description>
+ </property>
+ <property>
+ <name>discovery_zen_fd_ping_retries</name>
+ <value>5</value>
+ <description>Number of ping retries before blacklisting</description>
+ </property>
+ <!-- Gateway -->
+ <property>
+ <name>gateway_recover_after_data_nodes</name>
+ <value>3</value>
+ <description>Recover as long as this many data or master nodes have joined the cluster.</description>
+ </property>
+ <property>
+ <name>recover_after_time</name>
+ <value>15m</value>
+ <description>recover_after_time</description>
+ </property>
+ <property>
+ <name>expected_data_nodes</name>
+ <value>0</value>
+ <description>expected_data_nodes</description>
+ </property>
+ <!-- Index -->
+ <property>
+ <name>index_merge_scheduler_max_thread_count</name>
+ <value>5</value>
+ <description>index.merge.scheduler.max_thread_count</description>
+ </property>
+ <property>
+ <name>indices_memory_index_store_throttle_type</name>
+ <value>none</value>
+ <description>index_store_throttle_type</description>
+ </property>
+ <property>
+ <name>index_refresh_interval</name>
+ <value>1s</value>
+ <description>index refresh interval</description>
+ </property>
+ <property>
+ <name>index_translog_flush_threshold_size</name>
+ <value>5g</value>
+ <description>index_translog_flush_threshold_size</description>
+ </property>
+ <property>
+ <name>indices_memory_index_buffer_size</name>
+ <value>10%</value>
+ <description>Percentage of heap used for write buffers</description>
+ </property>
+ <property>
+ <name>bootstrap_memory_lock</name>
+ <value>true</value>
+ <description>The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out</description>
+ </property>
+ <property>
+ <name>threadpool_bulk_queue_size</name>
+ <value>3000</value>
+ <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request</description>
+ </property>
+ <property>
+ <name>threadpool_index_queue_size</name>
+ <value>1000</value>
+ <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request</description>
+ </property>
+ <property>
+ <name>indices_cluster_send_refresh_mapping</name>
+ <value>false</value>
+ <description>In order to make the index request more efficient, we have set this property on our data nodes</description>
+ </property>
+ <property>
+ <name>indices_fielddata_cache_size</name>
+ <value>25%</value>
+ <description>You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_disk_watermark_high</name>
+ <value>0.99</value>
+ <description>Property used when multiple drives are used to understand max thresholds</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_disk_threshold_enabled</name>
+ <value>true</value>
+ <description>Property used when multiple drives are used to understand if thresholding is active</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_disk_watermark_low</name>
+ <value>.97</value>
+ <description>Property used when multiple drives are used to understand min thresholds</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_node_concurrent_recoveries</name>
+ <value>4</value>
+ <description>Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart</description>
+ </property>
+ <property>
+ <name>network_host</name>
+ <value>[ _local_, _site_ ]</value>
+ <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description>
+ </property>
+ <property>
+ <name>network_publish_host</name>
+ <value>[]</value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
new file mode 100755
index 0000000..ea6ca38
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>elastic_home</name>
+ <value>/usr/share/elasticsearch/</value>
+ <description>Elasticsearch Home Directory</description>
+ </property>
+ <property>
+ <name>data_dir</name>
+ <value>/var/lib/elasticsearch/</value>
+ <description>Elasticsearch Data Directory</description>
+ </property>
+ <property>
+ <name>work_dir</name>
+ <value>/tmp/elasticsearch/</value>
+ <description>Elasticsearch Work Directory</description>
+ </property>
+ <property>
+ <name>conf_dir</name>
+ <value>/etc/elasticsearch/</value>
+ <description>Elasticsearch Configuration Directory</description>
+ </property>
+ <property>
+ <name>max_open_files</name>
+ <value>65536</value>
+ <description>Maximum number of open files</description>
+ </property>
+ <property>
+ <name>max_map_count</name>
+ <value>262144</value>
+ <description>Maximum number of memory map areas for process</description>
+ </property>
+
+ <!-- Elasticsearch sysconfig -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for elastic sysconfig file</description>
+ <value>
+# Directory where the Elasticsearch binary distribution resides
+ES_HOME={{elastic_home}}
+
+# Maximum number of open files
+MAX_OPEN_FILES={{max_open_files}}
+
+# Maximum number of VMA (Virtual Memory Areas) a process can own
+MAX_MAP_COUNT={{max_map_count}}
+
+# Elasticsearch log directory
+LOG_DIR={{log_dir}}
+
+# Elasticsearch data directory
+DATA_DIR={{data_dir}}
+
+# Elasticsearch work directory
+WORK_DIR={{work_dir}}
+
+# Elasticsearch conf directory
+CONF_DIR={{conf_dir}}
+
+# User to run as, change this to a specific elasticsearch user if possible
+# Also make sure, this user can write into the log directories in case you change them
+# This setting only works for the init script, but has to be configured separately for systemd startup
+ES_USER={{elastic_user}}
+
+# Elasticsearch pid directory
+PID_DIR={{pid_dir}}
+
+# JAVA_HOME must be provided here for OS that use systemd service launch
+JAVA_HOME={{java64_home}}
+
+# Additional Java options - now preferential to use 'jvm.options' file instead
+ES_JAVA_OPTS=""
+
+# https://www.elastic.co/guide/en/elasticsearch/reference/5.6/_memory_lock_check.html
+MAX_LOCKED_MEMORY=unlimited
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml
new file mode 100644
index 0000000..311e3c0
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>content</name>
+ <description>The jinja template for the Elasticsearch systemd override file. Applies only to platforms that use systemd.</description>
+ <value>
+[Service]
+LimitMEMLOCK=infinity
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
new file mode 100755
index 0000000..47abb45
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ELASTICSEARCH</name>
+ <displayName>Elasticsearch</displayName>
+ <comment>Indexing and Search</comment>
+ <version>5.6.2</version>
+ <components>
+ <component>
+ <name>ES_MASTER</name>
+ <displayName>Elasticsearch Master</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/elastic_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ <component>
+ <name>ES_SLAVE</name>
+ <displayName>Elasticsearch Data Node</displayName>
+ <category>SLAVE</category>
+ <cardinality>0+</cardinality>
+ <commandScript>
+ <script>scripts/elastic_slave.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat6</osFamily>
+ <packages>
+ <package>
+ <name>elasticsearch-5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat7</osFamily>
+ <packages>
+ <package>
+ <name>elasticsearch-5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>ubuntu14</osFamily>
+ <packages>
+ <package>
+ <name>elasticsearch=5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+ <configuration-dependencies>
+ <config-type>elastic-env</config-type>
+ <config-type>elastic-site</config-type>
+ <config-type>elastic-sysconfig</config-type>
+ <config-type>elastic-systemd</config-type>
+ <config-type>elastic-jvm-options</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
new file mode 100644
index 0000000..618d10a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
@@ -0,0 +1,266 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from ambari_commons.os_check import OSCheck
+from resource_management.core.exceptions import ExecutionFailed
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+from resource_management.core.resources import User
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.functions.get_user_call_output import get_user_call_output
+
+
+def service_check(cmd, user, label):
+ """
+ Executes a SysV service check command that adheres to LSB-compliant
+ return codes. The return codes are interpreted as defined
+ by the LSB.
+
+ See http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
+ for more information.
+
+ :param cmd: The service check command to execute.
+ :param label: The name of the service.
+ """
+ Logger.info("Performing service check; cmd={0}, user={1}, label={2}".format(cmd, user, label))
+ rc, out, err = get_user_call_output(cmd, user, is_checked_call=False)
+
+ if rc in [1, 2, 3]:
+ # if return code in [1, 2, 3], then 'program is not running' or 'program is dead'
+ Logger.info("{0} is not running".format(label))
+ raise ComponentIsNotRunning()
+
+ elif rc == 0:
+ # if return code = 0, then 'program is running or service is OK'
+ Logger.info("{0} is running".format(label))
+
+ else:
+ # else service state is unknown
+ err_msg = "{0} service check failed; cmd '{1}' returned {2}".format(label, cmd, rc)
+ Logger.error(err_msg)
+ raise ExecutionFailed(err_msg, rc, out, err)
+
+def is_systemd_running():
+ """
+ Determines if the platform is running Systemd.
+ :return True, if the platform is running Systemd. False, otherwise.
+ """
+ Logger.info("Is the platform running Systemd?")
+ rc, out, err = get_user_call_output("pidof systemd", "root", is_checked_call=False)
+ if rc == 0:
+ Logger.info("Systemd was found")
+ return True
+ else:
+ Logger.info("Systemd was NOT found")
+ return False
+
+def configure_systemd(params):
+ """
+ Configure Systemd for Elasticsearch.
+ """
+ Logger.info("Configuring Systemd for Elasticsearch");
+
+ # ensure the systemd directory for elasticsearch overrides exists
+ Logger.info("Create Systemd directory for Elasticsearch: {0}".format(params.systemd_elasticsearch_dir))
+ Directory(params.systemd_elasticsearch_dir,
+ create_parents=True,
+ owner='root',
+ group='root')
+
+ # when using Elasticsearch packages on systems that use systemd, system
+ # limits must also be specified via systemd.
+ # see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setting-system-settings.html#systemd
+ Logger.info("Elasticsearch systemd limits: {0}".format(params.systemd_override_file))
+ File(params.systemd_override_file,
+ content=InlineTemplate(params.systemd_override_template),
+ owner="root",
+ group="root")
+
+ # reload the configuration
+ Execute("systemctl daemon-reload")
+
+def create_user(params):
+ """
+ Creates the user required for Elasticsearch.
+ """
+ Logger.info("Creating user={0} in group={1}".format(params.elastic_user, params.elastic_group))
+ User(params.elastic_user, action = "create", groups = params.elastic_group)
+
+def create_directories(params, directories):
+ """
+ Creates one or more directories.
+ """
+ Logger.info("Creating directories: {0}".format(directories))
+ Directory(directories,
+ create_parents=True,
+ mode=0755,
+ owner=params.elastic_user,
+ group=params.elastic_group
+ )
+
+def create_elastic_env(params):
+ """
+ Creates the Elasticsearch environment file.
+ """
+ Logger.info("Create Elasticsearch environment file.")
+ File("{0}/elastic-env.sh".format(params.conf_dir),
+ owner=params.elastic_user,
+ group=params.elastic_group,
+ content=InlineTemplate(params.elastic_env_sh_template))
+
+def create_elastic_site(params, template_name):
+ """
+ Creates the Elasticsearch site file.
+ """
+ Logger.info("Creating Elasticsearch site file; template={0}".format(template_name))
+
+ elastic_site = params.config['configurations']['elastic-site']
+ path = "{0}/elasticsearch.yml".format(params.conf_dir)
+ template = Template(template_name, configurations=elastic_site)
+ File(path,
+ content=template,
+ owner=params.elastic_user,
+ group=params.elastic_group)
+
+def get_elastic_config_path(default="/etc/default/elasticsearch"):
+ """
+ Defines the path to the Elasticsearch environment file. This path will
+ differ based on the OS family.
+ :param default: The path used if the OS family is not recognized.
+ """
+ path = default
+ if OSCheck.is_redhat_family():
+ path = "/etc/sysconfig/elasticsearch"
+ elif OSCheck.is_ubuntu_family():
+ path = "/etc/default/elasticsearch"
+ else:
+ Logger.error("Unexpected OS family; using default path={0}".format(path))
+
+ return path
+
+def create_elastic_config(params):
+ """
+ Creates the Elasticsearch system config file. Usually lands at either
+ /etc/sysconfig/elasticsearch or /etc/default/elasticsearch.
+ """
+ path = get_elastic_config_path()
+ Logger.info("Creating the Elasticsearch system config; path={0}".format(path))
+ File(path, owner="root", group="root", content=InlineTemplate(params.sysconfig_template))
+
+def create_elastic_pam_limits(params):
+ """
+ Creates the PAM limits for Elasticsearch.
+ """
+ Logger.info("Creating Elasticsearch PAM limits.")
+
+ # in some OS this folder may not exist, so create it
+ Logger.info("Ensure PAM limits directory exists: {0}".format(params.limits_conf_dir))
+ Directory(params.limits_conf_dir,
+ create_parents=True,
+ owner='root',
+ group='root')
+
+ Logger.info("Creating Elasticsearch PAM limits; file={0}".format(params.limits_conf_file))
+ File(params.limits_conf_file,
+ content=Template('elasticsearch_limits.conf.j2'),
+ owner="root",
+ group="root")
+
+def create_elastic_jvm_options(params):
+ """
+ Defines the jvm.options file used to specify JVM options.
+ """
+ path = "{0}/jvm.options".format(params.conf_dir)
+ Logger.info("Creating Elasticsearch JVM Options; file={0}".format(path))
+ File(path,
+ content=InlineTemplate(params.jvm_options_template),
+ owner=params.elastic_user,
+ group=params.elastic_group)
+
+def get_data_directories(params):
+ """
+ Returns the directories to use for storing Elasticsearch data.
+ """
+ path = params.path_data
+ path = path.replace('"', '')
+ path = path.replace(' ', '')
+ path = path.split(',')
+ dirs = [p.replace('"', '') for p in path]
+
+ Logger.info("Elasticsearch data directories: dirs={0}".format(dirs))
+ return dirs
+
+def configure_master():
+ """
+ Configures the Elasticsearch master node.
+ """
+ import params
+
+ # define the directories required
+ dirs = [
+ params.log_dir,
+ params.pid_dir,
+ params.conf_dir,
+ "{0}/scripts".format(params.conf_dir)
+ ]
+ dirs += get_data_directories(params)
+
+ # configure the elasticsearch master
+ create_user(params)
+ create_directories(params, dirs)
+ create_elastic_env(params)
+ create_elastic_site(params, "elasticsearch.master.yaml.j2")
+ create_elastic_config(params)
+ create_elastic_pam_limits(params)
+ create_elastic_jvm_options(params)
+ if is_systemd_running():
+ configure_systemd(params)
+
+def configure_slave():
+ """
+ Configures the Elasticsearch slave node.
+ """
+ import params
+
+ # define the directories required
+ dirs = [
+ params.log_dir,
+ params.pid_dir,
+ params.conf_dir,
+ ]
+ dirs += get_data_directories(params)
+
+ # configure the elasticsearch slave
+ create_user(params)
+ create_directories(params, dirs)
+ create_elastic_env(params)
+ create_elastic_site(params, "elasticsearch.slave.yaml.j2")
+ create_elastic_config(params)
+ create_elastic_pam_limits(params)
+ create_elastic_jvm_options(params)
+ if is_systemd_running():
+ configure_systemd(params)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py
new file mode 100755
index 0000000..142ce4e
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py
@@ -0,0 +1,72 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core import shell
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.script import Script
+from resource_management.core.logger import Logger
+from elastic_commands import service_check
+from elastic_commands import configure_master
+
+class Elasticsearch(Script):
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+ Logger.info('Install Elasticsearch master node')
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+ Logger.info('Configure Elasticsearch master node')
+ configure_master()
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ Logger.info('Stop Elasticsearch master node')
+ Execute("service elasticsearch stop")
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ Logger.info('Start Elasticsearch master node')
+ self.configure(env)
+ Execute("service elasticsearch start")
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ Logger.info('Status check Elasticsearch master node')
+ service_check(
+ cmd="service elasticsearch status",
+ user=params.elastic_status_check_user,
+ label="Elasticsearch Master")
+
+ def restart(self, env):
+ import params
+ env.set_params(params)
+ Logger.info('Restart Elasticsearch master node')
+ self.configure(env)
+ Execute("service elasticsearch restart")
+
+
+if __name__ == "__main__":
+ Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py
new file mode 100755
index 0000000..2d559ff
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py
@@ -0,0 +1,71 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.script import Script
+
+from elastic_commands import service_check
+from elastic_commands import configure_slave
+
+class Elasticsearch(Script):
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+ Logger.info('Install Elasticsearch slave node')
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+ Logger.info('Configure Elasticsearch slave node')
+ configure_slave()
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ Logger.info('Stop Elasticsearch slave node')
+ Execute("service elasticsearch stop")
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ Execute("service elasticsearch start")
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ Logger.info('Status check Elasticsearch slave node')
+ service_check(
+ cmd="service elasticsearch status",
+ user=params.elastic_status_check_user,
+ label="Elasticsearch Slave")
+
+ def restart(self, env):
+ import params
+ env.set_params(params)
+ Logger.info('Restart Elasticsearch slave node')
+ self.configure(env)
+ Execute("service elasticsearch restart")
+
+
+if __name__ == "__main__":
+ Elasticsearch().execute()
[3/7] metron git commit: METRON-1462: Separate ES and Kibana from
Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Posted by mm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
deleted file mode 100755
index f0903ac..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from elasticsearch import Elasticsearch
-from elasticsearch.helpers import bulk
-import cPickle as pickle
-import argparse, sys, os.path
-import errno
-import os
-
-
-class DashboardIndex(object):
-
- def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs):
- """
- :arg host: hostname of the node (default: localhost)
- :arg port: port to use (integer, default: 9200)
- :arg url_prefix: optional url prefix for elasticsearch
- :arg timeout: default timeout in seconds (float, default: 10)
- """
- self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}])
-
- def get(self):
- """
- Get .kibana index from Elasticsearch
- """
- dotkibana = self.es.search(index='.kibana', size = 100)
- return dotkibana['hits']['hits']
-
- def load(self,filespec):
- """
- Save Index data on local filesystem
- :args filespec: path/filename for saved file
- """
- data=[]
- with open(filespec,'rb') as fp:
- data = pickle.load(fp)
- return data
-
- def save(self,filename,data):
- """
- Save Index data on local filesystem
- :args filespec: path/filename for saved file
- """
- with open(filename,'wb') as fp:
- pickle.dump(data,fp)
-
- def put(self,data):
- """
- Bulk write data to Elasticsearch
- :args data: data to be written (note: index name is specified in data)
- """
- bulk(self.es,data)
-
- def main(self,args):
-
- if args.save:
- print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec))
- self.save(filename=args.filespec,data=di.get())
- else:
- """
- Loads Kibana Dashboard definition from disk and replaces .kibana on index
- :args filespec: path/filename for saved file
- """
- if not os.path.isfile(args.filespec):
- raise IOError(
- errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
- self.es.indices.delete(index='.kibana', ignore=[400, 404])
- self.put(data=di.load(filespec=args.filespec))
-
-if __name__ == '__main__':
-
- parser = argparse.ArgumentParser()
- parser.add_argument("hostname", help="ES Hostname or IP", type=str)
- parser.add_argument("port", help="ES Port", type=int)
- parser.add_argument("filespec", help="file to be pushed from or saved to", type=str)
- parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true")
- args = parser.parse_args()
- di = DashboardIndex(host=args.hostname,port=args.port)
- di.main(args)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
deleted file mode 100644
index 6f38ed5..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
+++ /dev/null
@@ -1,233 +0,0 @@
-{
- "template" : ".kibana",
- "mappings" : {
- "search" : {
- "dynamic" : "strict",
- "properties" : {
- "columns" : {
- "type" : "keyword"
- },
- "description" : {
- "type" : "text"
- },
- "hits" : {
- "type" : "integer"
- },
- "kibanaSavedObjectMeta" : {
- "properties" : {
- "searchSourceJSON" : {
- "type" : "text"
- }
- }
- },
- "sort" : {
- "type" : "keyword"
- },
- "title" : {
- "type" : "text"
- },
- "version" : {
- "type" : "integer"
- }
- }
- },
- "url" : {
- "dynamic" : "strict",
- "properties" : {
- "accessCount" : {
- "type" : "long"
- },
- "accessDate" : {
- "type" : "date"
- },
- "createDate" : {
- "type" : "date"
- },
- "url" : {
- "type" : "text",
- "fields" : {
- "keyword" : {
- "type" : "keyword",
- "ignore_above" : 2048
- }
- }
- }
- }
- },
- "dashboard" : {
- "dynamic" : "strict",
- "properties" : {
- "description" : {
- "type" : "text"
- },
- "hits" : {
- "type" : "integer"
- },
- "kibanaSavedObjectMeta" : {
- "properties" : {
- "searchSourceJSON" : {
- "type" : "text"
- }
- }
- },
- "optionsJSON" : {
- "type" : "text"
- },
- "panelsJSON" : {
- "type" : "text"
- },
- "refreshInterval" : {
- "properties" : {
- "display" : {
- "type" : "keyword"
- },
- "pause" : {
- "type" : "boolean"
- },
- "section" : {
- "type" : "integer"
- },
- "value" : {
- "type" : "integer"
- }
- }
- },
- "timeFrom" : {
- "type" : "keyword"
- },
- "timeRestore" : {
- "type" : "boolean"
- },
- "timeTo" : {
- "type" : "keyword"
- },
- "title" : {
- "type" : "text"
- },
- "uiStateJSON" : {
- "type" : "text"
- },
- "version" : {
- "type" : "integer"
- }
- }
- },
- "index-pattern" : {
- "dynamic" : "strict",
- "properties" : {
- "fieldFormatMap" : {
- "type" : "text"
- },
- "fields" : {
- "type" : "text"
- },
- "intervalName" : {
- "type" : "keyword"
- },
- "notExpandable" : {
- "type" : "boolean"
- },
- "sourceFilters" : {
- "type" : "text"
- },
- "timeFieldName" : {
- "type" : "keyword"
- },
- "title" : {
- "type" : "text"
- }
- }
- },
- "timelion-sheet" : {
- "dynamic" : "strict",
- "properties" : {
- "description" : {
- "type" : "text"
- },
- "hits" : {
- "type" : "integer"
- },
- "kibanaSavedObjectMeta" : {
- "properties" : {
- "searchSourceJSON" : {
- "type" : "text"
- }
- }
- },
- "timelion_chart_height" : {
- "type" : "integer"
- },
- "timelion_columns" : {
- "type" : "integer"
- },
- "timelion_interval" : {
- "type" : "keyword"
- },
- "timelion_other_interval" : {
- "type" : "keyword"
- },
- "timelion_rows" : {
- "type" : "integer"
- },
- "timelion_sheet" : {
- "type" : "text"
- },
- "title" : {
- "type" : "text"
- },
- "version" : {
- "type" : "integer"
- }
- }
- },
- "visualization" : {
- "dynamic" : "strict",
- "properties" : {
- "description" : {
- "type" : "text"
- },
- "kibanaSavedObjectMeta" : {
- "properties" : {
- "searchSourceJSON" : {
- "type" : "text"
- }
- }
- },
- "savedSearchId" : {
- "type" : "keyword"
- },
- "title" : {
- "type" : "text"
- },
- "uiStateJSON" : {
- "type" : "text"
- },
- "version" : {
- "type" : "integer"
- },
- "visState" : {
- "type" : "text"
- }
- }
- },
- "server" : {
- "dynamic" : "strict",
- "properties" : {
- "uuid" : {
- "type" : "keyword"
- }
- }
- },
- "_default_" : {
- "dynamic" : "strict"
- },
- "config" : {
- "dynamic" : "true",
- "properties" : {
- "buildNum" : {
- "type" : "keyword"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
deleted file mode 100755
index a15f709..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
+++ /dev/null
@@ -1,119 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import errno
-import os
-
-from ambari_commons.os_check import OSCheck
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import Execute
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-from resource_management.libraries.functions.format import format as ambari_format
-from resource_management.libraries.script import Script
-
-from common import service_check
-
-class Kibana(Script):
-
- def install(self, env):
- import params
- env.set_params(params)
- Logger.info("Installing Kibana")
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- import params
- env.set_params(params)
- Logger.info("Configuring Kibana")
-
- directories = [params.log_dir, params.pid_dir, params.conf_dir]
- Directory(directories,
- mode=0755,
- owner=params.kibana_user,
- group=params.kibana_user
- )
-
- File("{0}/kibana.yml".format(params.conf_dir),
- owner=params.kibana_user,
- content=InlineTemplate(params.kibana_yml_template)
- )
-
- def stop(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- Logger.info("Stopping Kibana")
- Execute("service kibana stop")
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- self.configure(env)
- Logger.info("Starting Kibana")
- Execute("service kibana start")
-
- def restart(self, env):
- import params
- env.set_params(params)
- self.configure(env)
- Logger.info("Restarting Kibana")
- Execute("service kibana restart")
-
- def status(self, env):
- import params
- env.set_params(params)
- Logger.info('Status check Kibana')
- service_check("service kibana status", user=params.kibana_user, label="Kibana")
-
- @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
- def load_template(self, env):
- import params
- env.set_params(params)
-
- hostname = ambari_format("{es_host}")
- port = int(ambari_format("{es_port}"))
-
- Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname, port))
-
- kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template')
- if not os.path.isfile(kibanaTemplate):
- raise IOError(
- errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate)
-
- Logger.info("Loading .kibana index template from %s" % kibanaTemplate)
- template_cmd = ambari_format(
- 'curl -s -XPOST http://{es_host}:{es_port}/_template/.kibana -d @%s' % kibanaTemplate)
- Execute(template_cmd, logoutput=True)
-
- kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json')
- if not os.path.isfile(kibanaDashboardLoad):
- raise IOError(
- errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad)
-
- Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad)
-
- kibana_cmd = ambari_format(
- 'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_host}:{es_port}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad)
- Execute(kibana_cmd, logoutput=True)
-
-
-if __name__ == "__main__":
- Kibana().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
deleted file mode 100755
index ef4cb62..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Kibana Params configurations
-
-"""
-
-from urlparse import urlparse
-
-from resource_management.libraries.functions import format
-from resource_management.libraries.script import Script
-
-# server configurations
-config = Script.get_config()
-
-kibana_home = '/usr/share/kibana/'
-kibana_bin = '/usr/share/kibana/bin/'
-
-conf_dir = "/etc/kibana"
-kibana_user = config['configurations']['kibana-env']['kibana_user']
-kibana_group = config['configurations']['kibana-env']['kibana_group']
-log_dir = config['configurations']['kibana-env']['kibana_log_dir']
-pid_dir = config['configurations']['kibana-env']['kibana_pid_dir']
-pid_file = format("{pid_dir}/kibanasearch.pid")
-es_url = config['configurations']['kibana-env']['kibana_es_url']
-parsed = urlparse(es_url)
-es_host = parsed.netloc.split(':')[0]
-es_port = parsed.netloc.split(':')[1]
-kibana_port = config['configurations']['kibana-env']['kibana_server_port']
-kibana_server_host = config['configurations']['kibana-env']['kibana_server_host']
-kibana_default_application = config['configurations']['kibana-env']['kibana_default_application']
-hostname = config['hostname']
-java64_home = config['hostLevelParams']['java_home']
-kibana_yml_template = config['configurations']['kibana-site']['content']
-
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
deleted file mode 100755
index 448e102..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "metron_ui",
- "label": "Metron UI",
- "requires_user_name": "false",
- "component_name": "KIBANA_MASTER",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "kibana_server_port",
- "http_default_port": "5601",
- "https_property": "kibana_server_port",
- "https_default_port": "5601",
- "regex": "^(\\d+)$",
- "site": "kibana-env"
- }
- }
- ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 6a20af8..38bd94a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -211,6 +211,15 @@
</commandScript>
</customCommand>
<customCommand>
+ <name>KIBANA_DASHBOARD_INSTALL</name>
+ <background>false</background>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ <customCommand>
<name>ZEPPELIN_NOTEBOOK_IMPORT</name>
<commandScript>
<script>scripts/indexing_master.py</script>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py
new file mode 100755
index 0000000..8d2bad8
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
[6/7] metron git commit: METRON-1462: Separate ES and Kibana from
Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Posted by mm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
new file mode 100755
index 0000000..24f2306
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+
+def yamlify_variables(var) :
+ if isinstance(var, type(True)):
+ return str(var).lower()
+ else:
+ return var
+
+# server configurations
+config = Script.get_config()
+
+masters_also_are_datanodes = config['configurations']['elastic-site']['masters_also_are_datanodes']
+elastic_home = config['configurations']['elastic-sysconfig']['elastic_home']
+data_dir = config['configurations']['elastic-sysconfig']['data_dir']
+work_dir = config['configurations']['elastic-sysconfig']['work_dir']
+conf_dir = config['configurations']['elastic-sysconfig']['conf_dir']
+heap_size = config['configurations']['elastic-sysconfig']['heap_size']
+max_open_files = config['configurations']['elastic-sysconfig']['max_open_files']
+max_map_count = config['configurations']['elastic-sysconfig']['max_map_count']
+
+elastic_user = config['configurations']['elastic-env']['elastic_user']
+elastic_group = config['configurations']['elastic-env']['elastic_group']
+log_dir = config['configurations']['elastic-env']['elastic_log_dir']
+pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
+
+hostname = config['hostname']
+java64_home = config['hostLevelParams']['java_home']
+elastic_env_sh_template = config['configurations']['elastic-env']['content']
+sysconfig_template = config['configurations']['elastic-sysconfig']['content']
+
+cluster_name = config['configurations']['elastic-site']['cluster_name']
+zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts']
+
+path_data = config['configurations']['elastic-site']['path_data']
+http_cors_enabled = config['configurations']['elastic-site']['http_cors_enabled']
+http_port = config['configurations']['elastic-site']['http_port']
+transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port']
+
+recover_after_time = config['configurations']['elastic-site']['recover_after_time']
+gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes']
+expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes']
+index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count']
+index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size']
+index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval']
+indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type']
+index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards']
+index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas']
+indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size']
+bootstrap_memory_lock = yamlify_variables(config['configurations']['elastic-site']['bootstrap_memory_lock'])
+threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size']
+cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries']
+cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low']
+cluster_routing_allocation_disk_threshold_enabled = yamlify_variables(config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled'])
+cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high']
+indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size']
+indices_cluster_send_refresh_mapping = yamlify_variables(config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping'])
+threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size']
+
+discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout']
+discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval']
+discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout']
+discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries']
+
+network_host = config['configurations']['elastic-site']['network_host']
+network_publish_host = config['configurations']['elastic-site']['network_publish_host']
+
+limits_conf_dir = "/etc/security/limits.d"
+limits_conf_file = limits_conf_dir + "/elasticsearch.conf"
+elastic_user_nofile_limit = config['configurations']['elastic-env']['elastic_user_nofile_limit']
+elastic_user_nproc_limit = config['configurations']['elastic-env']['elastic_user_nproc_limit']
+elastic_user_memlock_soft_limit = config['configurations']['elastic-env']['elastic_user_memlock_soft_limit']
+elastic_user_memlock_hard_limit = config['configurations']['elastic-env']['elastic_user_memlock_hard_limit']
+
+# the status check (service elasticsearch status) cannot be run by the 'elasticsearch'
+# user due to the default permissions that are set when the package is installed. the
+# status check must be run as root
+elastic_status_check_user = 'root'
+
+# when using the RPM or Debian packages on systems that use systemd, system limits
+# must be specified via systemd.
+# see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setting-system-settings.html#systemd
+systemd_parent_dir = '/etc/systemd/system/'
+systemd_elasticsearch_dir = systemd_parent_dir + 'elasticsearch.service.d/'
+systemd_override_file = systemd_elasticsearch_dir + 'override.conf'
+systemd_override_template = config['configurations']['elastic-systemd']['content']
+
+heap_size = config['configurations']['elastic-jvm-options']['heap_size']
+jvm_options_template = config['configurations']['elastic-jvm-options']['content']
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py
new file mode 100755
index 0000000..ef9f6dd
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+
+
+def properties_inline_template(configurations):
+ return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }}
+{% endfor %}''', configurations_dict=configurations)
+
+
+def properties_config(filename, configurations=None, conf_dir=None,
+ mode=None, owner=None, group=None, brokerid=None):
+ config_content = properties_inline_template(configurations)
+ File(format("{conf_dir}/{filename}"), content=config_content, owner=owner,
+ group=group, mode=mode)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
new file mode 100755
index 0000000..3ac7c83
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+import subprocess
+import sys
+import re
+
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.script import Script
+from resource_management.core.logger import Logger
+
+class ServiceCheck(Script):
+
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ Logger.info("Running Elasticsearch service check")
+
+ port = self.get_port_from_range(params.http_port)
+ self.check_cluster_health(params.hostname, port)
+ self.index_document(params.hostname, port)
+
+ Logger.info("Elasticsearch service check successful")
+ exit(0)
+
+ def index_document(self, host, port, doc='{"name": "Ambari Service Check"}', index="ambari_service_check"):
+ """
+ Tests the health of Elasticsearch by indexing a document.
+
+ :param host: The name of a host running Elasticsearch.
+ :param port: The Elasticsearch HTTP port.
+ :param doc: The test document to put.
+ :param index: The name of the test index.
+ """
+ # put a document into a new index
+ Execute("curl -XPUT 'http://%s:%s/%s/test/1' -d '%s'" % (host, port, index, doc), logoutput=True)
+
+ # retrieve the document... use subprocess because we actually need the results here.
+ cmd_retrieve = "curl -XGET 'http://%s:%s/%s/test/1'" % (host, port, index)
+ proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+ response_retrieve = stdout
+ Logger.info("Retrieval response is: %s" % response_retrieve)
+ expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \
+ % (index, doc)
+
+ # delete the test index
+ cmd_delete = "curl -XDELETE 'http://%s:%s/%s'" % (host, port, index)
+ proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+ response_delete = stdout
+ Logger.info("Delete index response is: %s" % response_retrieve)
+ expected_delete = '{"acknowledged":true}'
+
+ if (expected_retrieve == response_retrieve) and (expected_delete == response_delete):
+ Logger.info("Successfully indexed document in Elasticsearch")
+ else:
+ Logger.info("Unable to retrieve document from Elasticsearch")
+ sys.exit(1)
+
+ def check_cluster_health(self, host, port, status="green", timeout="120s"):
+ """
+ Checks Elasticsearch cluster health. Will wait for a given health
+ state to be reached.
+
+ :param host: The name of a host running Elasticsearch.
+ :param port: The Elasticsearch HTTP port.
+ :param status: The expected cluster health state. By default, green.
+ :param timeout: How long to wait for the cluster. By default, 120 seconds.
+ """
+ Logger.info("Checking cluster health")
+
+ cmd = "curl -sS -XGET 'http://{0}:{1}/_cluster/health?wait_for_status={2}&timeout={3}' | grep '\"status\":\"{2}\"'"
+ Execute(cmd.format(host, port, status, timeout), logoutput=True, tries=5, try_sleep=10)
+
+ def get_port_from_range(self, port_range, delimiter="-", default="9200"):
+ """
+ Elasticsearch is configured with a range of ports to bind to, such as
+ 9200-9300. This function identifies a single port within the given range.
+
+ :param port_range: A range of ports that Elasticsearch binds to.
+ :param delimiter: The port range delimiter, by default "-".
+ :param default: If no port can be identified in the port_range, the default is returned.
+ :return A single port within the given range.
+ """
+ port = default
+ if delimiter in port_range:
+ ports = port_range.split(delimiter)
+ if len(ports) > 0:
+ port = ports[0]
+
+ return port
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
new file mode 100755
index 0000000..0629735
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+
+config = Script.get_config()
+
+elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
+elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid")
+elastic_user = config['configurations']['elastic-env']['elastic_user']
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2
new file mode 100755
index 0000000..8e20ba2
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2
@@ -0,0 +1,77 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+cluster:
+ name: {{cluster_name}}
+ routing:
+ allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
+ allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
+ allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
+ allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
+
+discovery:
+ zen:
+ ping:
+ unicast:
+ hosts: {{zen_discovery_ping_unicast_hosts}}
+
+node:
+ data: {{ masters_also_are_datanodes }}
+ master: true
+ name: {{hostname}}
+path:
+ data: {{path_data}}
+
+http:
+ port: {{http_port}}
+ cors.enabled: {{http_cors_enabled}}
+
+
+transport:
+ tcp:
+ port: {{transport_tcp_port}}
+
+gateway:
+ recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
+ recover_after_time: {{recover_after_time}}
+ expected_data_nodes: {{expected_data_nodes}}
+# https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html
+indices:
+ store.throttle.type: {{indices_memory_index_store_throttle_type}}
+ memory:
+ index_buffer_size: {{indices_memory_index_buffer_size}}
+ fielddata:
+ cache.size: {{indices_fielddata_cache_size}}
+
+bootstrap:
+ memory_lock: {{bootstrap_memory_lock}}
+ system_call_filter: false
+
+thread_pool:
+ bulk:
+ queue_size: {{threadpool_bulk_queue_size}}
+ index:
+ queue_size: {{threadpool_index_queue_size}}
+
+discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
+discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
+discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
+discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
+
+network.host: {{network_host}}
+network.publish_host: {{network_publish_host}}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2
new file mode 100755
index 0000000..6bf8399
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2
@@ -0,0 +1,78 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+cluster:
+ name: {{cluster_name}}
+ routing:
+ allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
+ allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
+ allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
+ allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
+
+discovery:
+ zen:
+ ping:
+ unicast:
+ hosts: {{zen_discovery_ping_unicast_hosts}}
+
+node:
+ data: true
+ master: false
+ name: {{hostname}}
+path:
+ data: {{path_data}}
+
+http:
+ port: {{http_port}}
+ cors.enabled: {{http_cors_enabled}}
+
+
+transport:
+ tcp:
+ port: {{transport_tcp_port}}
+
+gateway:
+ recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
+ recover_after_time: {{recover_after_time}}
+ expected_data_nodes: {{expected_data_nodes}}
+
+# https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html
+indices:
+ store.throttle.type: {{indices_memory_index_store_throttle_type}}
+ memory:
+ index_buffer_size: {{indices_memory_index_buffer_size}}
+ fielddata:
+ cache.size: {{indices_fielddata_cache_size}}
+
+bootstrap:
+ memory_lock: {{bootstrap_memory_lock}}
+ system_call_filter: false
+
+thread_pool:
+ bulk:
+ queue_size: {{threadpool_bulk_queue_size}}
+ index:
+ queue_size: {{threadpool_index_queue_size}}
+
+discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
+discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
+discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
+discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
+
+network.host: {{network_host}}
+network.publish_host: {{network_publish_host}}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2
new file mode 100644
index 0000000..99f72e1
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{elastic_user}} - nproc {{elastic_user_nproc_limit}}
+{{elastic_user}} - nofile {{elastic_user_nofile_limit}}
+{{elastic_user}} soft memlock {{elastic_user_memlock_soft_limit}}
+{{elastic_user}} hard memlock {{elastic_user_memlock_hard_limit}}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json
new file mode 100644
index 0000000..909828b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json
@@ -0,0 +1,43 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "es_health_link",
+ "label": "Elasticsearch Health",
+ "requires_user_name": "false",
+ "component_name": "ES_MASTER",
+ "url":"%@://%@:%@/_cat/health?v",
+ "port":{
+ "http_property": "http_port",
+ "http_default_port": "9200",
+ "https_property": "http_port",
+ "https_default_port": "9200",
+ "regex": "^(\\d+)",
+ "site": "elastic-site"
+ }
+ },
+ {
+ "name": "es_indices_link",
+ "label": "Elasticsearch Indexes",
+ "requires_user_name": "false",
+ "component_name": "ES_MASTER",
+ "url":"%@://%@:%@/_cat/indices?v",
+ "port":{
+ "http_property": "http_port",
+ "http_default_port": "9200",
+ "https_property": "http_port",
+ "https_default_port": "9200",
+ "regex": "^(\\d+)",
+ "site": "elastic-site"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json
new file mode 100755
index 0000000..130d018
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json
@@ -0,0 +1,8 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml
new file mode 100755
index 0000000..1246405
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>kibana_user</name>
+ <value>kibana</value>
+ <property-type>USER</property-type>
+ <description>Service User for Kibana</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ </property>
+ <property>
+ <name>kabana_group</name>
+ <value>kibana</value>
+ <property-type>GROUP</property-type>
+ <description>Service Group for Kibana</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ </property>
+ <property require-input="true">
+ <name>kibana_server_host</name>
+ <value>0.0.0.0</value>
+ <description>Host name or IP address that Kibana should bind to.</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_log_dir</name>
+ <value>/var/log/kibana</value>
+ <description>Log directory for Kibana</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_pid_dir</name>
+ <value>/var/run/kibana</value>
+ <description>PID directory for Kibana</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_es_url</name>
+ <value></value>
+ <description>The Elasticsearch instance to use for all your queries. (http://eshost:9200)</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_server_port</name>
+ <value>5000</value>
+ <description>Kibana back end server port to use.</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_default_application</name>
+ <value>default</value>
+ <description>The default application to load.</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml
new file mode 100755
index 0000000..d8d0513
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <!-- kibana.yml -->
+ <property>
+ <name>content</name>
+ <display-name>kibana.yml template</display-name>
+ <description>This is the jinja template for kibana.yml file</description>
+ <value>
+# Kibana is served by a back end server. This controls which port to use.
+server.port: {{ kibana_port }}
+
+# The host to bind the server to.
+# Kibana (like Elasticsearch) now binds to localhost for security purposes instead of 0.0.0.0 (all addresses). Previous binding to 0.0.0.0 also caused issues for Windows users.
+server.host: {{ kibana_server_host }}
+
+# If you are running kibana behind a proxy, and want to mount it at a path,
+# specify that path here. The basePath can't end in a slash.
+# server.basePath: ""
+
+# The maximum payload size in bytes on incoming server requests.
+# server.maxPayloadBytes: 1048576
+
+# The Elasticsearch instance to use for all your queries.
+elasticsearch.url: {{ es_url }}
+
+# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
+# then the host you use to connect to *this* Kibana instance will be sent.
+# elasticsearch.preserveHost: true
+
+# Kibana uses an index in Elasticsearch to store saved searches, visualizations
+# and dashboards. It will create a new index if it doesn't already exist.
+# kibana.index: ".kibana"
+
+# The default application to load.
+kibana.defaultAppId: "{{ kibana_default_application }}"
+
+# If your Elasticsearch is protected with basic auth, these are the user credentials
+# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana
+# users will still need to authenticate with Elasticsearch (which is proxied through
+# the Kibana server)
+# elasticsearch.username: "user"
+# elasticsearch.password: "pass"
+
+# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted)
+# server.ssl.cert: /path/to/your/server.crt
+# server.ssl.key: /path/to/your/server.key
+
+# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted)
+# elasticsearch.ssl.cert: /path/to/your/client.crt
+# elasticsearch.ssl.key: /path/to/your/client.key
+
+# If you need to provide a CA certificate for your Elasticsearch instance, put
+# the path of the pem file here.
+# elasticsearch.ssl.ca: /path/to/your/CA.pem
+
+# Set to false to have a complete disregard for the validity of the SSL
+# certificate.
+# elasticsearch.ssl.verify: true
+
+# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to
+# request_timeout setting
+# elasticsearch.pingTimeout: 1500
+
+# Time in milliseconds to wait for responses from the back end or elasticsearch.
+# This must be > 0
+# elasticsearch.requestTimeout: 30000
+
+# Time in milliseconds for Elasticsearch to wait for responses from shards.
+# Set to 0 to disable.
+# elasticsearch.shardTimeout: 0
+
+# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying
+# elasticsearch.startupTimeout: 5000
+
+# Set the path to where you would like the process id file to be created.
+# pid.file: /var/run/kibana.pid
+
+# If you would like to send the log output to a file you can set the path below.
+logging.dest: {{ log_dir }}/kibana.log
+
+# Set this to true to suppress all logging output.
+# logging.silent: false
+
+# Set this to true to suppress all logging output except for error messages.
+# logging.quiet: false
+
+# Set this to true to log all events, including system usage information and all requests.
+# logging.verbose: false
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
new file mode 100755
index 0000000..b542c54
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>KIBANA</name>
+ <displayName>Kibana</displayName>
+ <comment>Kibana Dashboard</comment>
+ <version>5.6.2</version>
+ <components>
+ <component>
+ <name>KIBANA_MASTER</name>
+ <displayName>Kibana Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <commandScript>
+ <script>scripts/kibana_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat6</osFamily>
+ <packages>
+ <package>
+ <name>python-elasticsearch</name>
+ </package>
+ <package>
+ <name>kibana-5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat7</osFamily>
+ <packages>
+ <package>
+ <name>python-elasticsearch</name>
+ </package>
+ <package>
+ <name>kibana-5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>ubuntu14</osFamily>
+ <packages>
+ <package>
+ <name>kibana=5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <configuration-dependencies>
+ <config-type>kibana-env</config-type>
+ <config-type>kibana-site</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
new file mode 100644
index 0000000..37100cd
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
@@ -0,0 +1,56 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.get_user_call_output import get_user_call_output
+from resource_management.core.exceptions import ExecutionFailed
+from resource_management.core.exceptions import ComponentIsNotRunning
+
+def service_check(cmd, user, label):
+ """
+ Executes a service check command that adheres to LSB-compliant
+ return codes. The return codes are interpreted as defined
+ by the LSB.
+
+ See http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
+ for more information.
+
+ :param cmd: The service check command to execute.
+ :param label: The name of the service.
+ """
+ Logger.info("Performing service check; cmd={0}, user={1}, label={2}".format(cmd, user, label))
+ rc, out, err = get_user_call_output(cmd, user, is_checked_call=False)
+
+ if len(err) > 0:
+ Logger.error(err)
+
+ if rc in [1, 2, 3]:
+ # if return code in [1, 2, 3], then 'program is not running' or 'program is dead'
+ Logger.info("{0} is not running".format(label))
+ raise ComponentIsNotRunning()
+
+ elif rc == 0:
+ # if return code = 0, then 'program is running or service is OK'
+ Logger.info("{0} is running".format(label))
+
+ else:
+ # else service state is unknown
+ err_msg = "{0} service check failed; cmd '{1}' returned {2}".format(label, cmd, rc)
+ Logger.error(err_msg)
+ raise ExecutionFailed(err_msg, rc, out, err)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
new file mode 100755
index 0000000..c013ea3
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
@@ -0,0 +1,81 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions.format import format as ambari_format
+from resource_management.libraries.script import Script
+
+from common import service_check
+
+class Kibana(Script):
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+ Logger.info("Installing Kibana")
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+ Logger.info("Configuring Kibana")
+
+ directories = [params.log_dir, params.pid_dir, params.conf_dir]
+ Directory(directories,
+ mode=0755,
+ owner=params.kibana_user,
+ group=params.kibana_user
+ )
+
+ File("{0}/kibana.yml".format(params.conf_dir),
+ owner=params.kibana_user,
+ content=InlineTemplate(params.kibana_yml_template)
+ )
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ Logger.info("Stopping Kibana")
+ Execute("service kibana stop")
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ Logger.info("Starting Kibana")
+ Execute("service kibana start")
+
+ def restart(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ Logger.info("Restarting Kibana")
+ Execute("service kibana restart")
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ Logger.info('Status check Kibana')
+ service_check("service kibana status", user=params.kibana_user, label="Kibana")
+
+if __name__ == "__main__":
+ Kibana().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
new file mode 100755
index 0000000..ef4cb62
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Kibana Params configurations
+
+"""
+
+from urlparse import urlparse
+
+from resource_management.libraries.functions import format
+from resource_management.libraries.script import Script
+
+# server configurations
+config = Script.get_config()
+
+kibana_home = '/usr/share/kibana/'
+kibana_bin = '/usr/share/kibana/bin/'
+
+conf_dir = "/etc/kibana"
+kibana_user = config['configurations']['kibana-env']['kibana_user']
+kibana_group = config['configurations']['kibana-env']['kibana_group']
+log_dir = config['configurations']['kibana-env']['kibana_log_dir']
+pid_dir = config['configurations']['kibana-env']['kibana_pid_dir']
+pid_file = format("{pid_dir}/kibanasearch.pid")
+es_url = config['configurations']['kibana-env']['kibana_es_url']
+parsed = urlparse(es_url)
+es_host = parsed.netloc.split(':')[0]
+es_port = parsed.netloc.split(':')[1]
+kibana_port = config['configurations']['kibana-env']['kibana_server_port']
+kibana_server_host = config['configurations']['kibana-env']['kibana_server_host']
+kibana_default_application = config['configurations']['kibana-env']['kibana_default_application']
+hostname = config['hostname']
+java64_home = config['hostLevelParams']['java_home']
+kibana_yml_template = config['configurations']['kibana-site']['content']
+
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
new file mode 100755
index 0000000..448e102
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "metron_ui",
+ "label": "Metron UI",
+ "requires_user_name": "false",
+ "component_name": "KIBANA_MASTER",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "kibana_server_port",
+ "http_default_port": "5601",
+ "https_property": "kibana_server_port",
+ "https_default_port": "5601",
+ "regex": "^(\\d+)$",
+ "site": "kibana-env"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json
new file mode 100644
index 0000000..1b5400c
--- /dev/null
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json
@@ -0,0 +1,76 @@
+{
+ "type": "full-release",
+ "name": "elasticsearch-ambari.mpack",
+ "version": "5.6.2",
+ "description": "Ambari Management Pack for Elasticsearch and Kibana",
+ "prerequisites": {
+ "min-ambari-version": "2.4.0.0",
+ "min-stack-versions": [
+ {
+ "stack_name": "HDP",
+ "stack_version": "2.3.0"
+ }
+ ]
+ },
+ "artifacts": [
+ {
+ "name": "ELASTICSEARCH-common-services",
+ "type" : "service-definitions",
+ "source_dir" : "common-services"
+ },
+ {
+ "name" : "ELASTICSEARCH-addon-services",
+ "type" : "stack-addon-service-definitions",
+ "source_dir": "addon-services",
+ "service_versions_map": [
+ {
+ "service_name" : "KIBANA",
+ "service_version" : "5.6.2",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.3"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.4"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.5"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.6"
+ }
+
+ ]
+ },
+ {
+ "service_name" : "ELASTICSEARCH",
+ "service_version" : "5.6.2",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.3"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.4"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.5"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.6"
+ }
+
+
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/README.md b/metron-deployment/packaging/ambari/metron-mpack/README.md
index 5179b5f..cd9399d 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/README.md
+++ b/metron-deployment/packaging/ambari/metron-mpack/README.md
@@ -86,15 +86,11 @@ Storm (and the Metron topologies) must be restarted after Metron is installed on
Kerberizing a cluster with a pre-existing Metron, automatically restarts all services during Kerberization. No additional manual restart is needed in this case.
-#### Zeppelin Import
-
-A custom action is available in Ambari to import Zeppelin dashboards. See the [metron-indexing documentation](../../../../metron-platform/metron-indexing) for more information.
-
#### Kibana Dashboards
The dashboards installed by the Kibana custom action are managed by two JSON files:
-* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
-* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
+* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
+* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json
The first file, `kibana.template`, is an Elasticsearch template that specifies the proper mapping types for the Kibana index. This configuration is necessary due to a bug
in the default dynamic mappings provided by Elasticsearch for long types versus integer that are incompatible with Kibana \[1\]. The second file, `dashboard-bulkload.json`,
@@ -102,7 +98,7 @@ contains all of the dashboard metadata necessary to create the Metron dashboard.
of documents necessary for setting up the dashboard in Elasticsearch. The main features installed are index patterns, searches, and a variety of visualizations
that are used in the Metron dashboard.
-Deploying the existing dashboard is easy. Once the MPack is installed, run the Kibana service's action "Load Template" to install dashboards. This will no longer overwrite
+Deploying the existing dashboard is easy. Once the MPack is installed, run the Metron service's action "Load Template" to install dashboards. This will no longer overwrite
the .kibana in Elasticsearch. The bulk load is configured to fail inserts for existing documents. If you want to _completely_ reload the dashboard, you would need to delete
the .kibana index and reload again from Ambari.
@@ -115,7 +111,7 @@ You can modify dashboards in Kibana and bring those changes into the core MPack
1. Export the .kibana index from ES
2. Convert the data into the ES bulk load format
-3. Replace the dashboard-bulkload.json file in the Kibana MPack.
+3. Replace the dashboard-bulkload.json file in the Metron MPack.
You can export the .kibana index using a tool like [https://github.com/taskrabbit/elasticsearch-dump](https://github.com/taskrabbit/elasticsearch-dump). The important
feature is to have one document per line. Here's an exmaple export using elasticsearch-dump
@@ -166,14 +162,14 @@ To create a new version of the file, make any necessary changes to Kibana (e.g.
**Saving a Backup**
```
-python packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py \
+python packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py \
$ES_HOST 9200 \
~/dashboard.p -s
```
**Restoring From a Backup**
```
-python packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py \
+python packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py \
$ES_HOST 9200 \
~/dashboard.p
```
@@ -181,6 +177,10 @@ $ES_HOST 9200 \
**Note**: This method of writing the Kibana dashboard to Elasticsearch will overwrite the entire .kibana index. Be sure to first backup the index first using either the new JSON
method described above, or writing out the dashboard.p pickle file using the old method (passing -s option to dashboardindex.py) described here.
+#### Zeppelin Import
+
+A custom action is available in Ambari to import Zeppelin dashboards. See the [metron-indexing documentation](../../../../metron-platform/metron-indexing) for more information.
+
#### Offline Installation
Retrieval of the GeoIP database is the only point during installation that reaches out to the internet. For an offline installation, the URL for the GeoIP database can be manually set to a local path on the file system such as `file:///home/root/geoip/GeoLite2-City.mmdb.gz`.
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
index ec4272e..81087a2 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
@@ -30,24 +30,10 @@
<filtered>true</filtered>
</fileSet>
<fileSet>
- <directory>src/main/resources/common-services</directory>
- <outputDirectory>common-services</outputDirectory>
- <excludes>
- <exclude>**/CURRENT/**</exclude>
- </excludes>
- </fileSet>
- <fileSet>
<directory>src/main/resources/addon-services/METRON/CURRENT</directory>
<outputDirectory>addon-services/METRON/${metron.version}</outputDirectory>
<filtered>true</filtered>
</fileSet>
- <fileSet>
- <directory>src/main/resources/addon-services</directory>
- <outputDirectory>addon-services</outputDirectory>
- <excludes>
- <exclude>**/CURRENT/**</exclude>
- </excludes>
- </fileSet>
</fileSets>
<files>
<file>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml
deleted file mode 100755
index accf7da..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>ELASTICSEARCH</name>
- <version>5.6.2</version>
- <extends>common-services/ELASTICSEARCH/5.6.2</extends>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
deleted file mode 100644
index ba21fb1..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<reposinfo>
- <os family="redhat6">
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
- <repoid>elasticsearch-5.x</repoid>
- <reponame>ELASTICSEARCH</reponame>
- </repo>
- </os>
- <os family="redhat7">
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
- <repoid>elasticsearch-5.x</repoid>
- <reponame>ELASTICSEARCH</reponame>
- </repo>
- </os>
- <os family="ubuntu14">
- <!--
- see details about Ambari fixes for Ubuntu introduced in Ambari 2.6+
- https://github.com/apache/ambari/commit/f8b29df9685b443d4a5c06c6e1725e4428c95b49#diff-6f26c26ed59462200d018c5e1e71e773
- https://issues.apache.org/jira/browse/AMBARI-21856
- -->
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/apt</baseurl>
- <repoid>elasticsearch-5.x</repoid>
- <reponame>ELASTICSEARCH</reponame>
- <distribution>stable</distribution>
- </repo>
- </os>
-</reposinfo>
-
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml
deleted file mode 100755
index 8a4fba2..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0"?>
-<!--
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>KIBANA</name>
- <version>5.6.2</version>
- <extends>common-services/KIBANA/5.6.2</extends>
- </service>
- </services>
-</metainfo>
-
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json
deleted file mode 100755
index 622a512..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "kibana_master_ui",
- "label": "Metron Dashboard",
- "requires_user_name": "false",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "kibana_server_port",
- "http_default_port": "5601",
- "https_property": "kibana_server_port",
- "https_default_port": "5601",
- "regex": "^(\\d+)$",
- "site": "kibana-env"
- }
- }
- ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
deleted file mode 100644
index 2755818..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
+++ /dev/null
@@ -1,60 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<reposinfo>
- <os family="redhat6">
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
- <repoid>kibana-5.x</repoid>
- <reponame>KIBANA</reponame>
- </repo>
- <repo>
- <baseurl>http://packages.elastic.co/curator/5/centos/6</baseurl>
- <repoid>ES-Curator-5.x</repoid>
- <reponame>CURATOR</reponame>
- </repo>
- </os>
- <os family="redhat7">
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
- <repoid>kibana-5.x</repoid>
- <reponame>KIBANA</reponame>
- </repo>
- <repo>
- <baseurl>http://packages.elastic.co/curator/5/centos/7</baseurl>
- <repoid>ES-Curator-5.x</repoid>
- <reponame>CURATOR</reponame>
- </repo>
- </os>
- <os family="ubuntu14">
- <!--
- see details about Ambari fixes for Ubuntu introduced in Ambari 2.6+
- https://github.com/apache/ambari/commit/f8b29df9685b443d4a5c06c6e1725e4428c95b49#diff-6f26c26ed59462200d018c5e1e71e773
- https://issues.apache.org/jira/browse/AMBARI-21856
- -->
- <repo>
- <baseurl>https://artifacts.elastic.co/packages/5.x/apt</baseurl>
- <repoid>kibana-5.x</repoid>
- <reponame>KIBANA</reponame>
- <distribution>stable</distribution>
- </repo>
- <repo>
- <baseurl>https://packages.elastic.co/curator/5/debian</baseurl>
- <repoid>ES-Curator-5.x</repoid>
- <reponame>CURATOR</reponame>
- <distribution>stable</distribution>
- </repo>
- </os>
-</reposinfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml
deleted file mode 100755
index 9e4f8ad..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml
+++ /dev/null
@@ -1,86 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>elastic_user</name>
- <value>elasticsearch</value>
- <property-type>USER</property-type>
- <description>Service user for Elasticsearch</description>
- <value-attributes>
- <type>user</type>
- <overridable>false</overridable>
- </value-attributes>
- </property>
- <property>
- <name>elastic_group</name>
- <value>elasticsearch</value>
- <property-type>GROUP</property-type>
- <description>Service group for Elasticsearch</description>
- <value-attributes>
- <type>user</type>
- <overridable>false</overridable>
- </value-attributes>
- </property>
- <property>
- <name>elastic_log_dir</name>
- <value>/var/log/elasticsearch</value>
- <description>Log directory for elastic</description>
- </property>
- <property>
- <name>elastic_pid_dir</name>
- <value>/var/run/elasticsearch</value>
- <description>The directory for pid files</description>
- </property>
- <!-- elasticsearch-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for elastic-env.sh file</description>
- <value>
-#!/bin/bash
-
-# Set ELASTICSEARCH specific environment variables here.
-
-# The java implementation to use.
-export JAVA_HOME={{java64_home}}
-export PATH=$PATH:$JAVA_HOME/bin
- </value>
- </property>
- <property>
- <name>elastic_user_nofile_limit</name>
- <value>65536</value>
- <description>Max open file limit for Elasticsearch user.</description>
- </property>
- <property>
- <name>elastic_user_nproc_limit</name>
- <value>2048</value>
- <description>Max number of processes for Elasticsearch user.</description>
- </property>
- <property>
- <name>elastic_user_memlock_soft_limit</name>
- <value>unlimited</value>
- <description>Max locked-in memory address space (soft memlock limit).</description>
- </property>
- <property>
- <name>elastic_user_memlock_hard_limit</name>
- <value>unlimited</value>
- <description>Max locked-in memory address space (hard memlock limit).</description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
deleted file mode 100644
index 5c6aaca..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
+++ /dev/null
@@ -1,144 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>heap_size</name>
- <value>512m</value>
- <description>JVM heap size</description>
- </property>
- <property>
- <name>content</name>
- <description>The jinja template for the Elasticsearch JVM options file.</description>
- <value>
-## JVM configuration
-
-################################################################
-## IMPORTANT: JVM heap size
-################################################################
-##
-## You should always set the min and max JVM heap
-## size to the same value. For example, to set
-## the heap to 4 GB, set:
-##
-## -Xms4g
-## -Xmx4g
-##
-## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
-## for more information
-##
-################################################################
-
-# Xms represents the initial size of total heap space
-# Xmx represents the maximum size of total heap space
-
--Xms{{heap_size}}
--Xmx{{heap_size}}
-
-################################################################
-## Expert settings
-################################################################
-##
-## All settings below this section are considered
-## expert settings. Don't tamper with them unless
-## you understand what you are doing
-##
-################################################################
-
-## GC configuration
--XX:+UseConcMarkSweepGC
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
-
-## optimizations
-
-# pre-touch memory pages used by the JVM during initialization
--XX:+AlwaysPreTouch
-
-## basic
-
-# force the server VM (remove on 32-bit client JVMs)
--server
-
-# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
--Xss1m
-
-# set to headless, just in case
--Djava.awt.headless=true
-
-# ensure UTF-8 encoding by default (e.g. filenames)
--Dfile.encoding=UTF-8
-
-# use our provided JNA always versus the system one
--Djna.nosys=true
-
-# use old-style file permissions on JDK9
--Djdk.io.permissionsUseCanonicalPath=true
-
-# flags to configure Netty
--Dio.netty.noUnsafe=true
--Dio.netty.noKeySetOptimization=true
--Dio.netty.recycler.maxCapacityPerThread=0
-
-# log4j 2
--Dlog4j.shutdownHookEnabled=false
--Dlog4j2.disable.jmx=true
--Dlog4j.skipJansi=true
-
-## heap dumps
-
-# generate a heap dump when an allocation from the Java heap fails
-# heap dumps are created in the working directory of the JVM
--XX:+HeapDumpOnOutOfMemoryError
-
-# specify an alternative path for heap dumps
-# ensure the directory exists and has sufficient space
-#-XX:HeapDumpPath=${heap.dump.path}
-
-## GC logging
-
-#-XX:+PrintGCDetails
-#-XX:+PrintGCTimeStamps
-#-XX:+PrintGCDateStamps
-#-XX:+PrintClassHistogram
-#-XX:+PrintTenuringDistribution
-#-XX:+PrintGCApplicationStoppedTime
-
-# log GC status to a file with time stamps
-# ensure the directory exists
-#-Xloggc:${loggc}
-
-# By default, the GC log file will not rotate.
-# By uncommenting the lines below, the GC log file
-# will be rotated every 128MB at most 32 times.
-#-XX:+UseGCLogFileRotation
-#-XX:NumberOfGCLogFiles=32
-#-XX:GCLogFileSize=128M
-
-# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
-# If documents were already indexed with unquoted fields in a previous version
-# of Elasticsearch, some operations may throw errors.
-#
-# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
-# only for migration purposes.
-#-Delasticsearch.json.allow_unquoted_field_names=true
- </value>
- </property>
-</configuration>
[4/7] metron git commit: METRON-1462: Separate ES and Kibana from
Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Posted by mm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
deleted file mode 100644
index 037f1c6..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
+++ /dev/null
@@ -1,88 +0,0 @@
-{ "create" : { "_id": "all-metron-index", "_type": "index-pattern" } }
-{"title":"*_index_*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\"
:true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",
\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false,
\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scri
pted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:ca\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:path_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"bro_timestamp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bro_timestamp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"capture_password\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFro
mDocValues\":true},{\"name\":\"certificate:exponent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:issuer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_length\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_after\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_be
fore\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:sig_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:version\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatab
le\":true,\"readFromDocValues\":true},{\"name\":\"client\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"command\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"compression_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_state\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_uids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"connect_info\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable
\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"data_channel:orig_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"depth\",\"type\":\"number\",\"cou
nt\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dgmlen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dhcp_host_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"direction\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dropped\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dst\",\"type\":\"ip\",\"count\"
:0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end-reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"end_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"end_reason.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\
":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\
"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchabl
e\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"dat
e\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_fields\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_hash\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"established\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethdst\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethlen\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethsrc\"
,\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"exception\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failed_sensor_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failure_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_desc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocV
alues\":true},{\"name\":\"filename\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"first_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"re
adFromDocValues\":true},{\"name\":\"helo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"history\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true
,\"readFromDocValues\":true},{\"name\":\"id\",\"type\":\"conflict\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false,\"conflictDescriptions\":{\"integer\":[\"snort_index_2017.11.06.19\",\"snort_index_2017.11.06.20\",\"snort_index_2017.11.06.21\",\"snort_index_2017.11.06.22\",\"snort_index_2017.11.06.23\",\"snort_index_2017.11.07.00\",\"snort_index_2017.11.07.01\"],\"keyword\":[\"bro_index_2017.11.02.23\",\"bro_index_2017.11.03.00\",\"bro_index_2017.11.03.01\",\"bro_index_2017.11.03.02\",\"bro_index_2017.11.03.03\",\"bro_index_2017.11.03.04\",\"bro_index_2017.11.03.13\",\"bro_index_2017.11.06.19\",\"bro_index_2017.11.06.20\",\"bro_index_2017.11.06.22\",\"bro_index_2017.11.06.23\",\"bro_index_2017.11.07.00\",\"bro_index_2017.11.07.01\"]}},{\"name\":\"iflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"in_reply_to\",\"type\":\"string\",\"count\":
0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"iplen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_alert\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_orig\",\"type\":\"boolean
\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_webmail\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"isn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"issuer_subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"kex_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"last_alert\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"last_reply\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"lease_ti
me\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"local_orig\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"local_resp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mac\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mac_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mailfrom\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"md5\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"
message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"method\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"missed_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"missing_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"msg_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}
,{\"name\":\"n\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"next_protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"note\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"notice\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"oct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\
"name\":\"orig_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"orig_fuids.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_ip_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"orig_mime_types.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_pkts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchab
le\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_string.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"overflow_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"parent_fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"password\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"path\",\"type\":\"string\",\"count\":0,\"s
cripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"peer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"peer_descr\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"pkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"port_num\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"proto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"protocol.keyword\",\"type\":\"string\",\"cou
nt\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qclass\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qclass_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qtype\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qtype_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"query\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message_bytes\",\"typ
e\":\"unknown\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"rcode\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcode_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcptto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"referrer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rejected\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"remote_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply
_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"request_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resp_fuids.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"read
FromDocValues\":true},{\"name\":\"resp_ip_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resp_mime_types.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_pkts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"response_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"result\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resumed\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,
\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"riflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"risn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"roct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rpkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ruflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchabl
e\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:dns\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:email\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:uri\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"second_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"seen_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sensor:type\",\"type\":\"string\",\"count\":0,\"scripte
d\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sensor:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"server\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"server_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"service\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sha1\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sha256\",\"type\":\"string\",\"cou
nt\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_generator\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_rev\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"sip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"software_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source:type\",\"type\":\
"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"source:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"src\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"src_peer\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"stack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"start_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status_code\",\"type
\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sub\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"suppress_for\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tcpack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpflags\",
\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpseq\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpwindow\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"threat:triage:level\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threat:triage:rules:0:score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threat:triage:score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,
\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timedout\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tls\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tos\",\"type\":\"num
ber\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"total_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"trans_depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"trans_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ttl\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tunnel_parents\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uid\",\"t
ype\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"unparsed_version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"uri\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user_agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"username\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"v
ersion:addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:major\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor2\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor3\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"x_originating_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"}
-{ "create" : { "_id": "AV-Sj0e2hKs1cXXnFMqF", "_type": "visualization" } }
-{"title":"Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a spri
ngboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
-{ "index" : { "_id": "5.6.2", "_type": "config" } }
-{"defaultIndex":"AV-S2e81hKs1cXXnFMqN"}
-{ "create" : { "_id": "AV-dVurck7f2nZ-iH3Ka", "_type": "visualization" } }
-{"title":"Event Count By Type","visState":"{\"title\":\"Event Count By Type\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"
times\":[],\"addTimeMarker\":false,\"type\":\"histogram\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"source:type\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"legendOpen\":true,\"colors\":{\"yaf\":\"#CCA300\",\"snort\":\"#C15C17\",\"bro\":\"#F9934E\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, yaf, snort\",\"params\":[\"bro\",\"yaf\",\"snort\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}},{\"match_phrase\":{\"source:type\":\"snort\"}}],\"minimum_sh
ould_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
-{ "create" : { "_id": "AV-YyJw3PfR7HJex-ZdY", "_type": "visualization" } }
-{"title":"All index TS event count","visState":"{\"title\":\"All index TS event count\",\"type\":\"metrics\",\"params\":{\"id\":\"eac7cbe0-c411-11e7-a0b9-2137696bd057\",\"type\":\"metric\",\"series\":[{\"id\":\"eac7cbe1-c411-11e7-a0b9-2137696bd057\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"eac7cbe2-c411-11e7-a0b9-2137696bd057\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Event Count\",\"split_filters\":[{\"color\":\"#68BC00\",\"id\":\"89be23f0-c4af-11e7-ac01-25d5c1ff2e49\"}],\"series_drop_last_bucket\":0}],\"time_field\":\"timestamp\",\"index_pattern\":\"bro_index*,snort_index*,yaf_index*\",\"interval\":\"1y\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"background_color_rules\":[{\"id\":\"022dc960-c412-11e7-a0b9-2137696bd057\"}],\"bar_color_rules\":[{\"id\":\"21ffb0
f0-c412-11e7-a0b9-2137696bd057\"}],\"filter\":\"\",\"drop_last_bucket\":0},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
-{ "create" : { "_id": "AV-cBm5JFLIoshSSHghu", "_type": "visualization" } }
-{"title":"All index TS Chart","visState":"{\"title\":\"All index TS Chart\",\"type\":\"metrics\",\"params\":{\"id\":\"eac7cbe0-c411-11e7-a0b9-2137696bd057\",\"type\":\"timeseries\",\"series\":[{\"id\":\"eac7cbe1-c411-11e7-a0b9-2137696bd057\",\"color\":\"rgba(0,156,224,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"eac7cbe2-c411-11e7-a0b9-2137696bd057\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":\"1\",\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Events\",\"terms_field\":\"source:type\",\"value_template\":\"{{value}}\"}],\"time_field\":\"timestamp\",\"index_pattern\":\"bro*,snort*,yaf*\",\"interval\":\"30s\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"background_color_rules\":[{\"id\":\"022dc960-c412-11e7-a0b9-2137696bd057\"}],\"bar_color_rules\":[{\"id\":\"21ffb0f0-c412-11e7-a0b9-2137696bd057\"}],\"show_grid\":1,\"drop_last_bucket\":0},\
"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
-{ "create" : { "_id": "AV-dXz9Lk7f2nZ-iH3Kb", "_type": "visualization" } }
-{"title":"Event Count Pie Chart","visState":"{\"title\":\"Event Count Pie Chart\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Events by Source Type\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source:type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_ph
rase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
-{ "create" : { "_id": "AV-ddhh7k7f2nZ-iH3Kx", "_type": "visualization" } }
-{"title":"Flow Location Map","visState":"{\"title\":\"Flow Location Map\",\"type\":\"tile_map\",\"params\":{\"mapType\":\"Scaled Circle Markers\",\"isDesaturated\":true,\"addTooltip\":true,\"heatMaxZoom\":0,\"heatMinOpacity\":0.1,\"heatRadius\":25,\"heatBlur\":15,\"legendPosition\":\"bottomright\",\"mapZoom\":2,\"mapCenter\":[0,0],\"wms\":{\"enabled\":false,\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\"options\":{\"version\":\"1.3.0\",\"layers\":\"0\",\"format\":\"image/png\",\"transparent\":true,\"attribution\":\"Maps provided by USGS\",\"styles\":\"\"}},\"type\":\"tile_map\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"geohash_grid\",\"schema\":\"segment\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:location_point\",\"autoPrecision\":true,\"useGeocentroid\":true,\"precision\":2,\"customLabel\":\"Flow Source Locations\"}}],\"listeners\"
:{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
-{ "create" : { "_id": "AV-dfk_gk7f2nZ-iH3K0", "_type": "visualization" } }
-{"title":"Events By Country","visState":"{\"title\":\"Events By Country\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:country\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type
\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
-{ "create" : { "_id": "AV-TUPlDgto7-W6O2b3n", "_type": "index-pattern" } }
-{"title":"yaf_index*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:t
s\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dip\",\"type\":\"string\",\"count\":0,\"s
cripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end-reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"end_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"end_reason.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:
ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichment
s:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFro
mDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"sea
rchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"iflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"i
p\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"isn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"oct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_string.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"pkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"prot
o\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"protocol.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"riflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"risn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"roct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rpkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":
\"rtag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ruflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"start_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tag\"
,\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"}
-{ "create" : { "_id": "AV-eebabk7f2nZ-iH3L1", "_type": "visualization" } }
-{"title":"YAF Flow Duration","visState":"{\"title\":\"YAF Flow Duration\",\"type\":\"area\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"Flow Duration (seconds)\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"
right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"area\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"duration\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Flow Duration (seconds)\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-TUPlDgto7-W6O2b3n\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
-{ "create" : { "_id": "AV-deDqXk7f2nZ-iH3Ky", "_type": "visualization" } }
-{"title":"Geo-IP Locations","visState":"{\"title\":\"Geo-IP Locations\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"60\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:country\",\"customLabel\":\"Unique Location(s)\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0
,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
-{ "create" : { "_id": "AV-YvG0DPfR7HJex-ZaS", "_type": "visualization" } }
-{"title":"Event Count","visState":"{\"title\":\"Event Count\",\"type\":\"metric\",\"params\":{\"addLegend\":false,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":100}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":false,\"width\":2},\"style\":{\"bgColor\":false,\"fontSize\":\"60\",\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"\"},\"type\":\"simple\",\"useRange\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Event Count\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedO
bjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
-{ "create" : { "_id": "AV-ejKEdk7f2nZ-iH3MI", "_type": "visualization" } }
-{"title":"Web Requests","visState":"{\"title\":\"Web Requests\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S
2e81hKs1cXXnFMqN\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"type\":\"phrases\",\"key\":\"protocol\",\"value\":\"http, https\",\"params\":[\"http\",\"https\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"protocol\":\"http\"}},{\"match_phrase\":{\"protocol\":\"https\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
-{ "create" : { "_id": "AV-ejbG6k7f2nZ-iH3MJ", "_type": "visualization" } }
-{"title":"DNS Requests","visState":"{\"title\":\"DNS Requests\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S
2e81hKs1cXXnFMqN\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"protocol\",\"value\":\"dns\"},\"query\":{\"match\":{\"protocol\":{\"query\":\"dns\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"}}
-{ "create" : { "_id": "AV-eh5Wgk7f2nZ-iH3MG", "_type": "visualization" } }
-{"title":"Snort Alert Types","visState":"{\"title\":\"Snort Alert Types\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"sig_id\",\"customLabel\":\"Alert Type(s)\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","ver
sion":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-TAoyPhKs1cXXnFMqi\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
-{ "create" : { "_id": "AV-ecrFkk7f2nZ-iH3L0", "_type": "visualization" } }
-{"title":"Yaf Flows Count","visState":"{\"title\":\"Yaf Flows Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":
\"AV-TUPlDgto7-W6O2b3n\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
-{ "create" : { "_id": "AV-ek_Jnk7f2nZ-iH3MK", "_type": "visualization" } }
-{"title":"Web Request Type","visState":"{\"title\":\"Web Request Type\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"method\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"type\":\"phrases\",\"key\":\"protocol\",\"value\":\"http, https\",\"params\":[\"http\",\"https\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"protocol\":\"http\"}},{\"match_phrase\":{\"protocol\":\"https\"}}],\"minimum_should_match\":1}},\
"$state\":{\"store\":\"appState\"}}]}"}}
-{ "create" : { "_id": "AV-S2e81hKs1cXXnFMqN", "_type": "index-pattern" } }
-{"title":"bro_index*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\
":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\"
,\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false
,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:ca\",\"type\":\"boolean\"
,\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:path_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"bro_timestamp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bro_timestamp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"capture_password\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:exponent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatabl
e\":true,\"readFromDocValues\":true},{\"name\":\"certificate:issuer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_length\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_after\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_before\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":
\"certificate:serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:sig_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:version\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"client\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggreg
atable\":true,\"readFromDocValues\":true},{\"name\":\"command\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"compression_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_state\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_uids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"connect_info\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchabl
e\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"data_channel:orig_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dhcp_host_name\",\"type\":\"st
ring\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"direction\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dropped\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dst\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues
\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\
"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"established\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failure_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_desc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDoc
Values\":true},{\"name\":\"file_mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"filename\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"first_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":tru
e,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"helo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"history\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregata
ble\":true,\"readFromDocValues\":true},{\"name\":\"host_p\",\"
<TRUNCATED>
[2/7] metron git commit: METRON-1462: Separate ES and Kibana from
Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Posted by mm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json
new file mode 100644
index 0000000..037f1c6
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json
@@ -0,0 +1,88 @@
+{ "create" : { "_id": "all-metron-index", "_type": "index-pattern" } }
+{"title":"*_index_*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\"
:true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",
\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false,
\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scri
pted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:ca\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:path_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"bro_timestamp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bro_timestamp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"capture_password\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFro
mDocValues\":true},{\"name\":\"certificate:exponent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:issuer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_length\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_after\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_be
fore\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:sig_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:version\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatab
le\":true,\"readFromDocValues\":true},{\"name\":\"client\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"command\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"compression_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_state\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_uids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"connect_info\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable
\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"data_channel:orig_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"depth\",\"type\":\"number\",\"cou
nt\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dgmlen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dhcp_host_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"direction\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dropped\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dst\",\"type\":\"ip\",\"count\"
:0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end-reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"end_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"end_reason.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\
":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\
"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchabl
e\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"dat
e\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_fields\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_hash\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"error_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"established\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethdst\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethlen\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ethsrc\"
,\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"exception\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failed_sensor_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failure_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_desc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocV
alues\":true},{\"name\":\"filename\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"first_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"re
adFromDocValues\":true},{\"name\":\"helo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"history\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true
,\"readFromDocValues\":true},{\"name\":\"id\",\"type\":\"conflict\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false,\"conflictDescriptions\":{\"integer\":[\"snort_index_2017.11.06.19\",\"snort_index_2017.11.06.20\",\"snort_index_2017.11.06.21\",\"snort_index_2017.11.06.22\",\"snort_index_2017.11.06.23\",\"snort_index_2017.11.07.00\",\"snort_index_2017.11.07.01\"],\"keyword\":[\"bro_index_2017.11.02.23\",\"bro_index_2017.11.03.00\",\"bro_index_2017.11.03.01\",\"bro_index_2017.11.03.02\",\"bro_index_2017.11.03.03\",\"bro_index_2017.11.03.04\",\"bro_index_2017.11.03.13\",\"bro_index_2017.11.06.19\",\"bro_index_2017.11.06.20\",\"bro_index_2017.11.06.22\",\"bro_index_2017.11.06.23\",\"bro_index_2017.11.07.00\",\"bro_index_2017.11.07.01\"]}},{\"name\":\"iflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"in_reply_to\",\"type\":\"string\",\"count\":
0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"iplen\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_alert\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_orig\",\"type\":\"boolean
\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"is_webmail\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"isn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"issuer_subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"kex_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"last_alert\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"last_reply\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"lease_ti
me\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"local_orig\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"local_resp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mac\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mac_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mailfrom\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"md5\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"
message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"method\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"missed_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"missing_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"msg_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}
,{\"name\":\"n\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"next_protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"note\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"notice\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"oct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\
"name\":\"orig_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"orig_fuids.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_ip_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"orig_mime_types.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"orig_pkts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchab
le\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_string.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"overflow_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"parent_fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"password\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"path\",\"type\":\"string\",\"count\":0,\"s
cripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"peer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"peer_descr\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"pkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"port_num\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"proto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"protocol.keyword\",\"type\":\"string\",\"cou
nt\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qclass\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qclass_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qtype\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"qtype_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"query\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"raw_message_bytes\",\"typ
e\":\"unknown\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"rcode\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcode_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rcptto\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"referrer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rejected\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"remote_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply
_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"reply_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"request_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resp_fuids.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"read
FromDocValues\":true},{\"name\":\"resp_ip_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_mime_types\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"resp_mime_types.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resp_pkts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"response_body_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"result\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"resumed\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,
\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"riflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"risn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"roct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rpkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ruflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchabl
e\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:dns\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:email\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"san:uri\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"second_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"seen_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sensor:type\",\"type\":\"string\",\"count\":0,\"scripte
d\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sensor:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"server\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"server_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"service\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sha1\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sha256\",\"type\":\"string\",\"cou
nt\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_generator\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sig_rev\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"sip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"software_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source:type\",\"type\":\
"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"source:type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"src\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"src_peer\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"stack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"start_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status_code\",\"type
\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"status_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sub\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"suppress_for\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tcpack\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpflags\",
\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpseq\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tcpwindow\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"threat:triage:level\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threat:triage:rules:0:score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threat:triage:score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,
\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timedout\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tls\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"tos\",\"type\":\"num
ber\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"total_bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"trans_depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"trans_id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ttl\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tunnel_parents\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uid\",\"t
ype\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"unparsed_version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"uri\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user_agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"username\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"v
ersion:addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:major\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor2\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version:minor3\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"x_originating_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"}
+{ "create" : { "_id": "AV-Sj0e2hKs1cXXnFMqF", "_type": "visualization" } }
+{"title":"Welcome to Apache Metron","visState":"{\"title\":\"Welcome to Apache Metron\",\"type\":\"markdown\",\"params\":{\"type\":\"markdown\",\"markdown\":\"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [\\n Snort](https://www.snort.org/), [\\n Bro](https://www.bro.org/), and [\\n YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the on-boarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\\n\\nApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a spri
ngboard upon which to create your own customized dashboards.\\n\\nThe panels below highlight the volume and variety of events that are currently being consumed by Apache Metron.\"},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "index" : { "_id": "5.6.2", "_type": "config" } }
+{"defaultIndex":"AV-S2e81hKs1cXXnFMqN"}
+{ "create" : { "_id": "AV-dVurck7f2nZ-iH3Ka", "_type": "visualization" } }
+{"title":"Event Count By Type","visState":"{\"title\":\"Event Count By Type\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"
times\":[],\"addTimeMarker\":false,\"type\":\"histogram\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"source:type\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"legendOpen\":true,\"colors\":{\"yaf\":\"#CCA300\",\"snort\":\"#C15C17\",\"bro\":\"#F9934E\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, yaf, snort\",\"params\":[\"bro\",\"yaf\",\"snort\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}},{\"match_phrase\":{\"source:type\":\"snort\"}}],\"minimum_sh
ould_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
+{ "create" : { "_id": "AV-YyJw3PfR7HJex-ZdY", "_type": "visualization" } }
+{"title":"All index TS event count","visState":"{\"title\":\"All index TS event count\",\"type\":\"metrics\",\"params\":{\"id\":\"eac7cbe0-c411-11e7-a0b9-2137696bd057\",\"type\":\"metric\",\"series\":[{\"id\":\"eac7cbe1-c411-11e7-a0b9-2137696bd057\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"eac7cbe2-c411-11e7-a0b9-2137696bd057\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Event Count\",\"split_filters\":[{\"color\":\"#68BC00\",\"id\":\"89be23f0-c4af-11e7-ac01-25d5c1ff2e49\"}],\"series_drop_last_bucket\":0}],\"time_field\":\"timestamp\",\"index_pattern\":\"bro_index*,snort_index*,yaf_index*\",\"interval\":\"1y\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"background_color_rules\":[{\"id\":\"022dc960-c412-11e7-a0b9-2137696bd057\"}],\"bar_color_rules\":[{\"id\":\"21ffb0
f0-c412-11e7-a0b9-2137696bd057\"}],\"filter\":\"\",\"drop_last_bucket\":0},\"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AV-cBm5JFLIoshSSHghu", "_type": "visualization" } }
+{"title":"All index TS Chart","visState":"{\"title\":\"All index TS Chart\",\"type\":\"metrics\",\"params\":{\"id\":\"eac7cbe0-c411-11e7-a0b9-2137696bd057\",\"type\":\"timeseries\",\"series\":[{\"id\":\"eac7cbe1-c411-11e7-a0b9-2137696bd057\",\"color\":\"rgba(0,156,224,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"eac7cbe2-c411-11e7-a0b9-2137696bd057\",\"type\":\"count\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":\"1\",\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Events\",\"terms_field\":\"source:type\",\"value_template\":\"{{value}}\"}],\"time_field\":\"timestamp\",\"index_pattern\":\"bro*,snort*,yaf*\",\"interval\":\"30s\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"background_color_rules\":[{\"id\":\"022dc960-c412-11e7-a0b9-2137696bd057\"}],\"bar_color_rules\":[{\"id\":\"21ffb0f0-c412-11e7-a0b9-2137696bd057\"}],\"show_grid\":1,\"drop_last_bucket\":0},\
"aggs\":[],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AV-dXz9Lk7f2nZ-iH3Kb", "_type": "visualization" } }
+{"title":"Event Count Pie Chart","visState":"{\"title\":\"Event Count Pie Chart\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Events by Source Type\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source:type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_ph
rase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
+{ "create" : { "_id": "AV-ddhh7k7f2nZ-iH3Kx", "_type": "visualization" } }
+{"title":"Flow Location Map","visState":"{\"title\":\"Flow Location Map\",\"type\":\"tile_map\",\"params\":{\"mapType\":\"Scaled Circle Markers\",\"isDesaturated\":true,\"addTooltip\":true,\"heatMaxZoom\":0,\"heatMinOpacity\":0.1,\"heatRadius\":25,\"heatBlur\":15,\"legendPosition\":\"bottomright\",\"mapZoom\":2,\"mapCenter\":[0,0],\"wms\":{\"enabled\":false,\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\"options\":{\"version\":\"1.3.0\",\"layers\":\"0\",\"format\":\"image/png\",\"transparent\":true,\"attribution\":\"Maps provided by USGS\",\"styles\":\"\"}},\"type\":\"tile_map\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"geohash_grid\",\"schema\":\"segment\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:location_point\",\"autoPrecision\":true,\"useGeocentroid\":true,\"precision\":2,\"customLabel\":\"Flow Source Locations\"}}],\"listeners\"
:{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
+{ "create" : { "_id": "AV-dfk_gk7f2nZ-iH3K0", "_type": "visualization" } }
+{"title":"Events By Country","visState":"{\"title\":\"Events By Country\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:country\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type
\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
+{ "create" : { "_id": "AV-TUPlDgto7-W6O2b3n", "_type": "index-pattern" } }
+{"title":"yaf_index*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:t
s\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dip\",\"type\":\"string\",\"count\":0,\"s
cripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end-reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"end_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"end_reason.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"end_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:
ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichment
s:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFro
mDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_src_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"sea
rchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"iflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr\",\"type\":\"ip\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"i
p\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"isn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"oct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_string\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_string.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"pkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"prot
o\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"protocol\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"protocol.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"riflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"risn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"roct\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rpkt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":
\"rtag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"rtt\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ruflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"start_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tag\"
,\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatinteljoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threatintelsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"uflags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]"}
+{ "create" : { "_id": "AV-eebabk7f2nZ-iH3L1", "_type": "visualization" } }
+{"title":"YAF Flow Duration","visState":"{\"title\":\"YAF Flow Duration\",\"type\":\"area\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"Flow Duration (seconds)\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"
right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"area\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"duration\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Flow Duration (seconds)\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-TUPlDgto7-W6O2b3n\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AV-deDqXk7f2nZ-iH3Ky", "_type": "visualization" } }
+{"title":"Geo-IP Locations","visState":"{\"title\":\"Geo-IP Locations\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"60\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"enrichments:geo:ip_src_addr:country\",\"customLabel\":\"Unique Location(s)\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0
,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
+{ "create" : { "_id": "AV-YvG0DPfR7HJex-ZaS", "_type": "visualization" } }
+{"title":"Event Count","visState":"{\"title\":\"Event Count\",\"type\":\"metric\",\"params\":{\"addLegend\":false,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":100}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":false,\"width\":2},\"style\":{\"bgColor\":false,\"fontSize\":\"60\",\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"\"},\"type\":\"simple\",\"useRange\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Event Count\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedO
bjectMeta":{"searchSourceJSON":"{\"index\":\"all-metron-index\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"all-metron-index\",\"type\":\"phrases\",\"key\":\"source:type\",\"value\":\"bro, snort, yaf\",\"params\":[\"bro\",\"snort\",\"yaf\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"source:type\":\"bro\"}},{\"match_phrase\":{\"source:type\":\"snort\"}},{\"match_phrase\":{\"source:type\":\"yaf\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
+{ "create" : { "_id": "AV-ejKEdk7f2nZ-iH3MI", "_type": "visualization" } }
+{"title":"Web Requests","visState":"{\"title\":\"Web Requests\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S
2e81hKs1cXXnFMqN\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"type\":\"phrases\",\"key\":\"protocol\",\"value\":\"http, https\",\"params\":[\"http\",\"https\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"protocol\":\"http\"}},{\"match_phrase\":{\"protocol\":\"https\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}]}"}}
+{ "create" : { "_id": "AV-ejbG6k7f2nZ-iH3MJ", "_type": "visualization" } }
+{"title":"DNS Requests","visState":"{\"title\":\"DNS Requests\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S
2e81hKs1cXXnFMqN\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"protocol\",\"value\":\"dns\"},\"query\":{\"match\":{\"protocol\":{\"query\":\"dns\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"}}
+{ "create" : { "_id": "AV-eh5Wgk7f2nZ-iH3MG", "_type": "visualization" } }
+{"title":"Snort Alert Types","visState":"{\"title\":\"Snort Alert Types\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"sig_id\",\"customLabel\":\"Alert Type(s)\"}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","ver
sion":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-TAoyPhKs1cXXnFMqi\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AV-ecrFkk7f2nZ-iH3L0", "_type": "visualization" } }
+{"title":"Yaf Flows Count","visState":"{\"title\":\"Yaf Flows Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":60,\"bgColor\":false,\"labelColor\":false,\"subText\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":
\"AV-TUPlDgto7-W6O2b3n\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}
+{ "create" : { "_id": "AV-ek_Jnk7f2nZ-iH3MK", "_type": "visualization" } }
+{"title":"Web Request Type","visState":"{\"title\":\"Web Request Type\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"type\":\"pie\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"method\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AV-S2e81hKs1cXXnFMqN\",\"type\":\"phrases\",\"key\":\"protocol\",\"value\":\"http, https\",\"params\":[\"http\",\"https\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"protocol\":\"http\"}},{\"match_phrase\":{\"protocol\":\"https\"}}],\"minimum_should_match\":1}},\
"$state\":{\"store\":\"appState\"}}]}"}}
+{ "create" : { "_id": "AV-S2e81hKs1cXXnFMqN", "_type": "index-pattern" } }
+{"title":"bro_index*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\
":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\"
,\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false
,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:ca\",\"type\":\"boolean\"
,\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"basic_constraints:path_len\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"bro_timestamp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bro_timestamp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"capture_password\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:exponent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatabl
e\":true,\"readFromDocValues\":true},{\"name\":\"certificate:issuer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_length\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:key_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_after\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:not_valid_before\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":
\"certificate:serial\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:sig_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"certificate:version\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cipher_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"client\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggreg
atable\":true,\"readFromDocValues\":true},{\"name\":\"command\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"compression_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_state\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"conn_uids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"connect_info\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"curve\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchabl
e\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"data_channel:orig_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_h\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data_channel:resp_p\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"depth\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dhcp_host_name\",\"type\":\"st
ring\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"direction\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dropped\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dst\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentjoinbolt:joiner:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:city\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues
\":true},{\"name\":\"enrichments:geo:ip_dst_addr:country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:dmaCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:latitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:locID\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:location_point\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:longitude\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\
"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichments:geo:ip_dst_addr:postalCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"enrichmentsplitterbolt:splitter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"established\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"failure_reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_desc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDoc
Values\":true},{\"name\":\"file_mime_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"file_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"filename\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"first_received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"fuid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"fuids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":tru
e,\"readFromDocValues\":true},{\"name\":\"guid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"guid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"helo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"history\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_key_alg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregata
ble\":true,\"readFromDocValues\":true},{\"name\":\"host_p\
<TRUNCATED>
[5/7] metron git commit: METRON-1462: Separate ES and Kibana from
Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Posted by mm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
deleted file mode 100755
index 34df1e4..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
+++ /dev/null
@@ -1,198 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Elastic search Configurations -->
-
-<configuration supports_final="true">
- <!-- Configurations -->
- <property>
- <name>cluster_name</name>
- <value>metron</value>
- <description>Elasticsearch Cluster Name identifies your Elasticsearch subsystem</description>
- </property>
- <property>
- <name>masters_also_are_datanodes</name>
- <value>"false"</value>
- <description>ES Masters and Slaves cannot be installed on the same nodes. Set this to "true" if you want the ES master nodes to serve as combined master/datanodes. Note: surround value in quotes.</description>
- <value-attributes>
- <type>string</type>
- </value-attributes>
- </property>
- <property>
- <name>zen_discovery_ping_unicast_hosts</name>
- <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.-->
- <!--Also need to document whether should list masters only, or all ES nodes. I think this one is all nodes, but previous inline comment said Masters.-->
- <value></value>
- <description>Unicast discovery list of hosts to act as gossip routers, comma-separated list with square brackets: [ eshost1, eshost2 ]</description>
- </property>
- <property>
- <name>index_number_of_shards</name>
- <value>4</value>
- <description>Set the number of shards (splits) of an index. Changes are not effective after index creation. Usually set to 1 for single-node install.</description>
- </property>
- <property>
- <name>index_number_of_replicas</name>
- <value>2</value>
- <description>Set the number of replicas (copies in addition to the first) of an index. Usually set to 0 for single-node install.</description>
- </property>
- <property>
- <name>path_data</name>
- <value>"/opt/lmm/es_data"</value>
- <description>Comma-separated list of directories where to store index data allocated for each node: "/mnt/first","/mnt/second". Number of paths should relate to number of shards, and preferably should be on separate physical volumes.</description>
- </property>
- <property>
- <name>http_cors_enabled</name>
- <value>"false"</value>
- <description>Enable or disable cross-origin resource sharing, i.e. whether a browser on another origin can do requests to Elasticsearch. Defaults to false.</description>
- <value-attributes>
- <type>string</type>
- </value-attributes>
- </property>
- <property>
- <name>http_port</name>
- <value>9200-9300</value>
- <description>Set a custom port to listen for HTTP traffic</description>
- </property>
- <property>
- <name>transport_tcp_port</name>
- <value>9300-9400</value>
- <description>Set a custom port for the node to node communication</description>
- </property>
- <!-- Multi-node Discovery -->
- <property>
- <name>discovery_zen_ping_timeout</name>
- <value>3s</value>
- <description>Wait for ping responses for master discovery</description>
- </property>
- <property>
- <name>discovery_zen_fd_ping_interval</name>
- <value>15s</value>
- <description>Wait for ping for cluster discovery</description>
- </property>
- <property>
- <name>discovery_zen_fd_ping_timeout</name>
- <value>60s</value>
- <description>Wait for ping for cluster discovery</description>
- </property>
- <property>
- <name>discovery_zen_fd_ping_retries</name>
- <value>5</value>
- <description>Number of ping retries before blacklisting</description>
- </property>
- <!-- Gateway -->
- <property>
- <name>gateway_recover_after_data_nodes</name>
- <value>3</value>
- <description>Recover as long as this many data or master nodes have joined the cluster.</description>
- </property>
- <property>
- <name>recover_after_time</name>
- <value>15m</value>
- <description>recover_after_time</description>
- </property>
- <property>
- <name>expected_data_nodes</name>
- <value>0</value>
- <description>expected_data_nodes</description>
- </property>
- <!-- Index -->
- <property>
- <name>index_merge_scheduler_max_thread_count</name>
- <value>5</value>
- <description>index.merge.scheduler.max_thread_count</description>
- </property>
- <property>
- <name>indices_memory_index_store_throttle_type</name>
- <value>none</value>
- <description>index_store_throttle_type</description>
- </property>
- <property>
- <name>index_refresh_interval</name>
- <value>1s</value>
- <description>index refresh interval</description>
- </property>
- <property>
- <name>index_translog_flush_threshold_size</name>
- <value>5g</value>
- <description>index_translog_flush_threshold_size</description>
- </property>
- <property>
- <name>indices_memory_index_buffer_size</name>
- <value>10%</value>
- <description>Percentage of heap used for write buffers</description>
- </property>
- <property>
- <name>bootstrap_memory_lock</name>
- <value>true</value>
- <description>The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out</description>
- </property>
- <property>
- <name>threadpool_bulk_queue_size</name>
- <value>3000</value>
- <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request</description>
- </property>
- <property>
- <name>threadpool_index_queue_size</name>
- <value>1000</value>
- <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request</description>
- </property>
- <property>
- <name>indices_cluster_send_refresh_mapping</name>
- <value>false</value>
- <description>In order to make the index request more efficient, we have set this property on our data nodes</description>
- </property>
- <property>
- <name>indices_fielddata_cache_size</name>
- <value>25%</value>
- <description>You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index</description>
- </property>
- <property>
- <name>cluster_routing_allocation_disk_watermark_high</name>
- <value>0.99</value>
- <description>Property used when multiple drives are used to understand max thresholds</description>
- </property>
- <property>
- <name>cluster_routing_allocation_disk_threshold_enabled</name>
- <value>true</value>
- <description>Property used when multiple drives are used to understand if thresholding is active</description>
- </property>
- <property>
- <name>cluster_routing_allocation_disk_watermark_low</name>
- <value>.97</value>
- <description>Property used when multiple drives are used to understand min thresholds</description>
- </property>
- <property>
- <name>cluster_routing_allocation_node_concurrent_recoveries</name>
- <value>4</value>
- <description>Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart</description>
- </property>
- <property>
- <name>network_host</name>
- <value>[ _local_, _site_ ]</value>
- <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description>
- </property>
- <property>
- <name>network_publish_host</name>
- <value>[]</value>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
deleted file mode 100755
index ea6ca38..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
+++ /dev/null
@@ -1,97 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>elastic_home</name>
- <value>/usr/share/elasticsearch/</value>
- <description>Elasticsearch Home Directory</description>
- </property>
- <property>
- <name>data_dir</name>
- <value>/var/lib/elasticsearch/</value>
- <description>Elasticsearch Data Directory</description>
- </property>
- <property>
- <name>work_dir</name>
- <value>/tmp/elasticsearch/</value>
- <description>Elasticsearch Work Directory</description>
- </property>
- <property>
- <name>conf_dir</name>
- <value>/etc/elasticsearch/</value>
- <description>Elasticsearch Configuration Directory</description>
- </property>
- <property>
- <name>max_open_files</name>
- <value>65536</value>
- <description>Maximum number of open files</description>
- </property>
- <property>
- <name>max_map_count</name>
- <value>262144</value>
- <description>Maximum number of memory map areas for process</description>
- </property>
-
- <!-- Elasticsearch sysconfig -->
- <property>
- <name>content</name>
- <description>This is the jinja template for elastic sysconfig file</description>
- <value>
-# Directory where the Elasticsearch binary distribution resides
-ES_HOME={{elastic_home}}
-
-# Maximum number of open files
-MAX_OPEN_FILES={{max_open_files}}
-
-# Maximum number of VMA (Virtual Memory Areas) a process can own
-MAX_MAP_COUNT={{max_map_count}}
-
-# Elasticsearch log directory
-LOG_DIR={{log_dir}}
-
-# Elasticsearch data directory
-DATA_DIR={{data_dir}}
-
-# Elasticsearch work directory
-WORK_DIR={{work_dir}}
-
-# Elasticsearch conf directory
-CONF_DIR={{conf_dir}}
-
-# User to run as, change this to a specific elasticsearch user if possible
-# Also make sure, this user can write into the log directories in case you change them
-# This setting only works for the init script, but has to be configured separately for systemd startup
-ES_USER={{elastic_user}}
-
-# Elasticsearch pid directory
-PID_DIR={{pid_dir}}
-
-# JAVA_HOME must be provided here for OS that use systemd service launch
-JAVA_HOME={{java64_home}}
-
-# Additional Java options - now preferential to use 'jvm.options' file instead
-ES_JAVA_OPTS=""
-
-# https://www.elastic.co/guide/en/elasticsearch/reference/5.6/_memory_lock_check.html
-MAX_LOCKED_MEMORY=unlimited
- </value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml
deleted file mode 100644
index 311e3c0..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>content</name>
- <description>The jinja template for the Elasticsearch systemd override file. Applies only to platforms that use systemd.</description>
- <value>
-[Service]
-LimitMEMLOCK=infinity
- </value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
deleted file mode 100755
index 47abb45..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
+++ /dev/null
@@ -1,97 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>ELASTICSEARCH</name>
- <displayName>Elasticsearch</displayName>
- <comment>Indexing and Search</comment>
- <version>5.6.2</version>
- <components>
- <component>
- <name>ES_MASTER</name>
- <displayName>Elasticsearch Master</displayName>
- <category>MASTER</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/elastic_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- <component>
- <name>ES_SLAVE</name>
- <displayName>Elasticsearch Data Node</displayName>
- <category>SLAVE</category>
- <cardinality>0+</cardinality>
- <commandScript>
- <script>scripts/elastic_slave.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
- <osSpecifics>
- <osSpecific>
- <osFamily>redhat6</osFamily>
- <packages>
- <package>
- <name>elasticsearch-5.6.2</name>
- </package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>redhat7</osFamily>
- <packages>
- <package>
- <name>elasticsearch-5.6.2</name>
- </package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>ubuntu14</osFamily>
- <packages>
- <package>
- <name>elasticsearch=5.6.2</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
- <configuration-dependencies>
- <config-type>elastic-env</config-type>
- <config-type>elastic-site</config-type>
- <config-type>elastic-sysconfig</config-type>
- <config-type>elastic-systemd</config-type>
- <config-type>elastic-jvm-options</config-type>
- </configuration-dependencies>
- <restartRequiredAfterChange>true</restartRequiredAfterChange>
- <quickLinksConfigurations>
- <quickLinksConfiguration>
- <fileName>quicklinks.json</fileName>
- <default>true</default>
- </quickLinksConfiguration>
- </quickLinksConfigurations>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
deleted file mode 100644
index 618d10a..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
+++ /dev/null
@@ -1,266 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-
-from ambari_commons.os_check import OSCheck
-from resource_management.core.exceptions import ExecutionFailed
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.resources.system import Execute
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-from resource_management.core.source import Template
-from resource_management.core.resources import User
-from resource_management.core.logger import Logger
-from resource_management.libraries.functions import format as ambari_format
-from resource_management.libraries.functions.get_user_call_output import get_user_call_output
-
-
-def service_check(cmd, user, label):
- """
- Executes a SysV service check command that adheres to LSB-compliant
- return codes. The return codes are interpreted as defined
- by the LSB.
-
- See http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
- for more information.
-
- :param cmd: The service check command to execute.
- :param label: The name of the service.
- """
- Logger.info("Performing service check; cmd={0}, user={1}, label={2}".format(cmd, user, label))
- rc, out, err = get_user_call_output(cmd, user, is_checked_call=False)
-
- if rc in [1, 2, 3]:
- # if return code in [1, 2, 3], then 'program is not running' or 'program is dead'
- Logger.info("{0} is not running".format(label))
- raise ComponentIsNotRunning()
-
- elif rc == 0:
- # if return code = 0, then 'program is running or service is OK'
- Logger.info("{0} is running".format(label))
-
- else:
- # else service state is unknown
- err_msg = "{0} service check failed; cmd '{1}' returned {2}".format(label, cmd, rc)
- Logger.error(err_msg)
- raise ExecutionFailed(err_msg, rc, out, err)
-
-def is_systemd_running():
- """
- Determines if the platform is running Systemd.
- :return True, if the platform is running Systemd. False, otherwise.
- """
- Logger.info("Is the platform running Systemd?")
- rc, out, err = get_user_call_output("pidof systemd", "root", is_checked_call=False)
- if rc == 0:
- Logger.info("Systemd was found")
- return True
- else:
- Logger.info("Systemd was NOT found")
- return False
-
-def configure_systemd(params):
- """
- Configure Systemd for Elasticsearch.
- """
- Logger.info("Configuring Systemd for Elasticsearch");
-
- # ensure the systemd directory for elasticsearch overrides exists
- Logger.info("Create Systemd directory for Elasticsearch: {0}".format(params.systemd_elasticsearch_dir))
- Directory(params.systemd_elasticsearch_dir,
- create_parents=True,
- owner='root',
- group='root')
-
- # when using Elasticsearch packages on systems that use systemd, system
- # limits must also be specified via systemd.
- # see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setting-system-settings.html#systemd
- Logger.info("Elasticsearch systemd limits: {0}".format(params.systemd_override_file))
- File(params.systemd_override_file,
- content=InlineTemplate(params.systemd_override_template),
- owner="root",
- group="root")
-
- # reload the configuration
- Execute("systemctl daemon-reload")
-
-def create_user(params):
- """
- Creates the user required for Elasticsearch.
- """
- Logger.info("Creating user={0} in group={1}".format(params.elastic_user, params.elastic_group))
- User(params.elastic_user, action = "create", groups = params.elastic_group)
-
-def create_directories(params, directories):
- """
- Creates one or more directories.
- """
- Logger.info("Creating directories: {0}".format(directories))
- Directory(directories,
- create_parents=True,
- mode=0755,
- owner=params.elastic_user,
- group=params.elastic_group
- )
-
-def create_elastic_env(params):
- """
- Creates the Elasticsearch environment file.
- """
- Logger.info("Create Elasticsearch environment file.")
- File("{0}/elastic-env.sh".format(params.conf_dir),
- owner=params.elastic_user,
- group=params.elastic_group,
- content=InlineTemplate(params.elastic_env_sh_template))
-
-def create_elastic_site(params, template_name):
- """
- Creates the Elasticsearch site file.
- """
- Logger.info("Creating Elasticsearch site file; template={0}".format(template_name))
-
- elastic_site = params.config['configurations']['elastic-site']
- path = "{0}/elasticsearch.yml".format(params.conf_dir)
- template = Template(template_name, configurations=elastic_site)
- File(path,
- content=template,
- owner=params.elastic_user,
- group=params.elastic_group)
-
-def get_elastic_config_path(default="/etc/default/elasticsearch"):
- """
- Defines the path to the Elasticsearch environment file. This path will
- differ based on the OS family.
- :param default: The path used if the OS family is not recognized.
- """
- path = default
- if OSCheck.is_redhat_family():
- path = "/etc/sysconfig/elasticsearch"
- elif OSCheck.is_ubuntu_family():
- path = "/etc/default/elasticsearch"
- else:
- Logger.error("Unexpected OS family; using default path={0}".format(path))
-
- return path
-
-def create_elastic_config(params):
- """
- Creates the Elasticsearch system config file. Usually lands at either
- /etc/sysconfig/elasticsearch or /etc/default/elasticsearch.
- """
- path = get_elastic_config_path()
- Logger.info("Creating the Elasticsearch system config; path={0}".format(path))
- File(path, owner="root", group="root", content=InlineTemplate(params.sysconfig_template))
-
-def create_elastic_pam_limits(params):
- """
- Creates the PAM limits for Elasticsearch.
- """
- Logger.info("Creating Elasticsearch PAM limits.")
-
- # in some OS this folder may not exist, so create it
- Logger.info("Ensure PAM limits directory exists: {0}".format(params.limits_conf_dir))
- Directory(params.limits_conf_dir,
- create_parents=True,
- owner='root',
- group='root')
-
- Logger.info("Creating Elasticsearch PAM limits; file={0}".format(params.limits_conf_file))
- File(params.limits_conf_file,
- content=Template('elasticsearch_limits.conf.j2'),
- owner="root",
- group="root")
-
-def create_elastic_jvm_options(params):
- """
- Defines the jvm.options file used to specify JVM options.
- """
- path = "{0}/jvm.options".format(params.conf_dir)
- Logger.info("Creating Elasticsearch JVM Options; file={0}".format(path))
- File(path,
- content=InlineTemplate(params.jvm_options_template),
- owner=params.elastic_user,
- group=params.elastic_group)
-
-def get_data_directories(params):
- """
- Returns the directories to use for storing Elasticsearch data.
- """
- path = params.path_data
- path = path.replace('"', '')
- path = path.replace(' ', '')
- path = path.split(',')
- dirs = [p.replace('"', '') for p in path]
-
- Logger.info("Elasticsearch data directories: dirs={0}".format(dirs))
- return dirs
-
-def configure_master():
- """
- Configures the Elasticsearch master node.
- """
- import params
-
- # define the directories required
- dirs = [
- params.log_dir,
- params.pid_dir,
- params.conf_dir,
- "{0}/scripts".format(params.conf_dir)
- ]
- dirs += get_data_directories(params)
-
- # configure the elasticsearch master
- create_user(params)
- create_directories(params, dirs)
- create_elastic_env(params)
- create_elastic_site(params, "elasticsearch.master.yaml.j2")
- create_elastic_config(params)
- create_elastic_pam_limits(params)
- create_elastic_jvm_options(params)
- if is_systemd_running():
- configure_systemd(params)
-
-def configure_slave():
- """
- Configures the Elasticsearch slave node.
- """
- import params
-
- # define the directories required
- dirs = [
- params.log_dir,
- params.pid_dir,
- params.conf_dir,
- ]
- dirs += get_data_directories(params)
-
- # configure the elasticsearch slave
- create_user(params)
- create_directories(params, dirs)
- create_elastic_env(params)
- create_elastic_site(params, "elasticsearch.slave.yaml.j2")
- create_elastic_config(params)
- create_elastic_pam_limits(params)
- create_elastic_jvm_options(params)
- if is_systemd_running():
- configure_systemd(params)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py
deleted file mode 100755
index 142ce4e..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py
+++ /dev/null
@@ -1,72 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core import shell
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.script import Script
-from resource_management.core.logger import Logger
-from elastic_commands import service_check
-from elastic_commands import configure_master
-
-class Elasticsearch(Script):
-
- def install(self, env):
- import params
- env.set_params(params)
- Logger.info('Install Elasticsearch master node')
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- import params
- env.set_params(params)
- Logger.info('Configure Elasticsearch master node')
- configure_master()
-
- def stop(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- Logger.info('Stop Elasticsearch master node')
- Execute("service elasticsearch stop")
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- Logger.info('Start Elasticsearch master node')
- self.configure(env)
- Execute("service elasticsearch start")
-
- def status(self, env):
- import params
- env.set_params(params)
- Logger.info('Status check Elasticsearch master node')
- service_check(
- cmd="service elasticsearch status",
- user=params.elastic_status_check_user,
- label="Elasticsearch Master")
-
- def restart(self, env):
- import params
- env.set_params(params)
- Logger.info('Restart Elasticsearch master node')
- self.configure(env)
- Execute("service elasticsearch restart")
-
-
-if __name__ == "__main__":
- Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py
deleted file mode 100755
index 2d559ff..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py
+++ /dev/null
@@ -1,71 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.script import Script
-
-from elastic_commands import service_check
-from elastic_commands import configure_slave
-
-class Elasticsearch(Script):
-
- def install(self, env):
- import params
- env.set_params(params)
- Logger.info('Install Elasticsearch slave node')
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- import params
- env.set_params(params)
- Logger.info('Configure Elasticsearch slave node')
- configure_slave()
-
- def stop(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- Logger.info('Stop Elasticsearch slave node')
- Execute("service elasticsearch stop")
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- self.configure(env)
- Execute("service elasticsearch start")
-
- def status(self, env):
- import params
- env.set_params(params)
- Logger.info('Status check Elasticsearch slave node')
- service_check(
- cmd="service elasticsearch status",
- user=params.elastic_status_check_user,
- label="Elasticsearch Slave")
-
- def restart(self, env):
- import params
- env.set_params(params)
- Logger.info('Restart Elasticsearch slave node')
- self.configure(env)
- Execute("service elasticsearch restart")
-
-
-if __name__ == "__main__":
- Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
deleted file mode 100755
index 24f2306..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script import Script
-
-def yamlify_variables(var) :
- if isinstance(var, type(True)):
- return str(var).lower()
- else:
- return var
-
-# server configurations
-config = Script.get_config()
-
-masters_also_are_datanodes = config['configurations']['elastic-site']['masters_also_are_datanodes']
-elastic_home = config['configurations']['elastic-sysconfig']['elastic_home']
-data_dir = config['configurations']['elastic-sysconfig']['data_dir']
-work_dir = config['configurations']['elastic-sysconfig']['work_dir']
-conf_dir = config['configurations']['elastic-sysconfig']['conf_dir']
-heap_size = config['configurations']['elastic-sysconfig']['heap_size']
-max_open_files = config['configurations']['elastic-sysconfig']['max_open_files']
-max_map_count = config['configurations']['elastic-sysconfig']['max_map_count']
-
-elastic_user = config['configurations']['elastic-env']['elastic_user']
-elastic_group = config['configurations']['elastic-env']['elastic_group']
-log_dir = config['configurations']['elastic-env']['elastic_log_dir']
-pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
-
-hostname = config['hostname']
-java64_home = config['hostLevelParams']['java_home']
-elastic_env_sh_template = config['configurations']['elastic-env']['content']
-sysconfig_template = config['configurations']['elastic-sysconfig']['content']
-
-cluster_name = config['configurations']['elastic-site']['cluster_name']
-zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts']
-
-path_data = config['configurations']['elastic-site']['path_data']
-http_cors_enabled = config['configurations']['elastic-site']['http_cors_enabled']
-http_port = config['configurations']['elastic-site']['http_port']
-transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port']
-
-recover_after_time = config['configurations']['elastic-site']['recover_after_time']
-gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes']
-expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes']
-index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count']
-index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size']
-index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval']
-indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type']
-index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards']
-index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas']
-indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size']
-bootstrap_memory_lock = yamlify_variables(config['configurations']['elastic-site']['bootstrap_memory_lock'])
-threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size']
-cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries']
-cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low']
-cluster_routing_allocation_disk_threshold_enabled = yamlify_variables(config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled'])
-cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high']
-indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size']
-indices_cluster_send_refresh_mapping = yamlify_variables(config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping'])
-threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size']
-
-discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout']
-discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval']
-discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout']
-discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries']
-
-network_host = config['configurations']['elastic-site']['network_host']
-network_publish_host = config['configurations']['elastic-site']['network_publish_host']
-
-limits_conf_dir = "/etc/security/limits.d"
-limits_conf_file = limits_conf_dir + "/elasticsearch.conf"
-elastic_user_nofile_limit = config['configurations']['elastic-env']['elastic_user_nofile_limit']
-elastic_user_nproc_limit = config['configurations']['elastic-env']['elastic_user_nproc_limit']
-elastic_user_memlock_soft_limit = config['configurations']['elastic-env']['elastic_user_memlock_soft_limit']
-elastic_user_memlock_hard_limit = config['configurations']['elastic-env']['elastic_user_memlock_hard_limit']
-
-# the status check (service elasticsearch status) cannot be run by the 'elasticsearch'
-# user due to the default permissions that are set when the package is installed. the
-# status check must be run as root
-elastic_status_check_user = 'root'
-
-# when using the RPM or Debian packages on systems that use systemd, system limits
-# must be specified via systemd.
-# see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setting-system-settings.html#systemd
-systemd_parent_dir = '/etc/systemd/system/'
-systemd_elasticsearch_dir = systemd_parent_dir + 'elasticsearch.service.d/'
-systemd_override_file = systemd_elasticsearch_dir + 'override.conf'
-systemd_override_template = config['configurations']['elastic-systemd']['content']
-
-heap_size = config['configurations']['elastic-jvm-options']['heap_size']
-jvm_options_template = config['configurations']['elastic-jvm-options']['content']
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py
deleted file mode 100755
index ef9f6dd..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-
-
-def properties_inline_template(configurations):
- return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }}
-{% endfor %}''', configurations_dict=configurations)
-
-
-def properties_config(filename, configurations=None, conf_dir=None,
- mode=None, owner=None, group=None, brokerid=None):
- config_content = properties_inline_template(configurations)
- File(format("{conf_dir}/{filename}"), content=config_content, owner=owner,
- group=group, mode=mode)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
deleted file mode 100755
index 3ac7c83..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from __future__ import print_function
-
-import subprocess
-import sys
-import re
-
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.script import Script
-from resource_management.core.logger import Logger
-
-class ServiceCheck(Script):
-
- def service_check(self, env):
- import params
- env.set_params(params)
- Logger.info("Running Elasticsearch service check")
-
- port = self.get_port_from_range(params.http_port)
- self.check_cluster_health(params.hostname, port)
- self.index_document(params.hostname, port)
-
- Logger.info("Elasticsearch service check successful")
- exit(0)
-
- def index_document(self, host, port, doc='{"name": "Ambari Service Check"}', index="ambari_service_check"):
- """
- Tests the health of Elasticsearch by indexing a document.
-
- :param host: The name of a host running Elasticsearch.
- :param port: The Elasticsearch HTTP port.
- :param doc: The test document to put.
- :param index: The name of the test index.
- """
- # put a document into a new index
- Execute("curl -XPUT 'http://%s:%s/%s/test/1' -d '%s'" % (host, port, index, doc), logoutput=True)
-
- # retrieve the document... use subprocess because we actually need the results here.
- cmd_retrieve = "curl -XGET 'http://%s:%s/%s/test/1'" % (host, port, index)
- proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
- response_retrieve = stdout
- Logger.info("Retrieval response is: %s" % response_retrieve)
- expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \
- % (index, doc)
-
- # delete the test index
- cmd_delete = "curl -XDELETE 'http://%s:%s/%s'" % (host, port, index)
- proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
- response_delete = stdout
- Logger.info("Delete index response is: %s" % response_retrieve)
- expected_delete = '{"acknowledged":true}'
-
- if (expected_retrieve == response_retrieve) and (expected_delete == response_delete):
- Logger.info("Successfully indexed document in Elasticsearch")
- else:
- Logger.info("Unable to retrieve document from Elasticsearch")
- sys.exit(1)
-
- def check_cluster_health(self, host, port, status="green", timeout="120s"):
- """
- Checks Elasticsearch cluster health. Will wait for a given health
- state to be reached.
-
- :param host: The name of a host running Elasticsearch.
- :param port: The Elasticsearch HTTP port.
- :param status: The expected cluster health state. By default, green.
- :param timeout: How long to wait for the cluster. By default, 120 seconds.
- """
- Logger.info("Checking cluster health")
-
- cmd = "curl -sS -XGET 'http://{0}:{1}/_cluster/health?wait_for_status={2}&timeout={3}' | grep '\"status\":\"{2}\"'"
- Execute(cmd.format(host, port, status, timeout), logoutput=True, tries=5, try_sleep=10)
-
- def get_port_from_range(self, port_range, delimiter="-", default="9200"):
- """
- Elasticsearch is configured with a range of ports to bind to, such as
- 9200-9300. This function identifies a single port within the given range.
-
- :param port_range: A range of ports that Elasticsearch binds to.
- :param delimiter: The port range delimiter, by default "-".
- :param default: If no port can be identified in the port_range, the default is returned.
- :return A single port within the given range.
- """
- port = default
- if delimiter in port_range:
- ports = port_range.split(delimiter)
- if len(ports) > 0:
- port = ports[0]
-
- return port
-
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
deleted file mode 100755
index 0629735..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script import Script
-
-config = Script.get_config()
-
-elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
-elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid")
-elastic_user = config['configurations']['elastic-env']['elastic_user']
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2
deleted file mode 100755
index 8e20ba2..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2
+++ /dev/null
@@ -1,77 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-cluster:
- name: {{cluster_name}}
- routing:
- allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
- allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
- allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
- allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
-
-discovery:
- zen:
- ping:
- unicast:
- hosts: {{zen_discovery_ping_unicast_hosts}}
-
-node:
- data: {{ masters_also_are_datanodes }}
- master: true
- name: {{hostname}}
-path:
- data: {{path_data}}
-
-http:
- port: {{http_port}}
- cors.enabled: {{http_cors_enabled}}
-
-
-transport:
- tcp:
- port: {{transport_tcp_port}}
-
-gateway:
- recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
- recover_after_time: {{recover_after_time}}
- expected_data_nodes: {{expected_data_nodes}}
-# https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html
-indices:
- store.throttle.type: {{indices_memory_index_store_throttle_type}}
- memory:
- index_buffer_size: {{indices_memory_index_buffer_size}}
- fielddata:
- cache.size: {{indices_fielddata_cache_size}}
-
-bootstrap:
- memory_lock: {{bootstrap_memory_lock}}
- system_call_filter: false
-
-thread_pool:
- bulk:
- queue_size: {{threadpool_bulk_queue_size}}
- index:
- queue_size: {{threadpool_index_queue_size}}
-
-discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
-discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
-discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
-discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
-
-network.host: {{network_host}}
-network.publish_host: {{network_publish_host}}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2
deleted file mode 100755
index 6bf8399..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2
+++ /dev/null
@@ -1,78 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-cluster:
- name: {{cluster_name}}
- routing:
- allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
- allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
- allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
- allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
-
-discovery:
- zen:
- ping:
- unicast:
- hosts: {{zen_discovery_ping_unicast_hosts}}
-
-node:
- data: true
- master: false
- name: {{hostname}}
-path:
- data: {{path_data}}
-
-http:
- port: {{http_port}}
- cors.enabled: {{http_cors_enabled}}
-
-
-transport:
- tcp:
- port: {{transport_tcp_port}}
-
-gateway:
- recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
- recover_after_time: {{recover_after_time}}
- expected_data_nodes: {{expected_data_nodes}}
-
-# https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html
-indices:
- store.throttle.type: {{indices_memory_index_store_throttle_type}}
- memory:
- index_buffer_size: {{indices_memory_index_buffer_size}}
- fielddata:
- cache.size: {{indices_fielddata_cache_size}}
-
-bootstrap:
- memory_lock: {{bootstrap_memory_lock}}
- system_call_filter: false
-
-thread_pool:
- bulk:
- queue_size: {{threadpool_bulk_queue_size}}
- index:
- queue_size: {{threadpool_index_queue_size}}
-
-discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
-discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
-discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
-discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
-
-network.host: {{network_host}}
-network.publish_host: {{network_publish_host}}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2
deleted file mode 100644
index 99f72e1..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{{elastic_user}} - nproc {{elastic_user_nproc_limit}}
-{{elastic_user}} - nofile {{elastic_user_nofile_limit}}
-{{elastic_user}} soft memlock {{elastic_user_memlock_soft_limit}}
-{{elastic_user}} hard memlock {{elastic_user_memlock_hard_limit}}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json
deleted file mode 100644
index 909828b..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "es_health_link",
- "label": "Elasticsearch Health",
- "requires_user_name": "false",
- "component_name": "ES_MASTER",
- "url":"%@://%@:%@/_cat/health?v",
- "port":{
- "http_property": "http_port",
- "http_default_port": "9200",
- "https_property": "http_port",
- "https_default_port": "9200",
- "regex": "^(\\d+)",
- "site": "elastic-site"
- }
- },
- {
- "name": "es_indices_link",
- "label": "Elasticsearch Indexes",
- "requires_user_name": "false",
- "component_name": "ES_MASTER",
- "url":"%@://%@:%@/_cat/indices?v",
- "port":{
- "http_property": "http_port",
- "http_default_port": "9200",
- "https_property": "http_port",
- "https_default_port": "9200",
- "regex": "^(\\d+)",
- "site": "elastic-site"
- }
- }
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json
deleted file mode 100755
index 130d018..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "_comment" : "Record format:",
- "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps" : {
- "_comment" : "dependencies for all cases",
- "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml
deleted file mode 100755
index 1246405..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-env.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
- <property>
- <name>kibana_user</name>
- <value>kibana</value>
- <property-type>USER</property-type>
- <description>Service User for Kibana</description>
- <value-attributes>
- <type>user</type>
- <overridable>false</overridable>
- </value-attributes>
- </property>
- <property>
- <name>kabana_group</name>
- <value>kibana</value>
- <property-type>GROUP</property-type>
- <description>Service Group for Kibana</description>
- <value-attributes>
- <type>user</type>
- <overridable>false</overridable>
- </value-attributes>
- </property>
- <property require-input="true">
- <name>kibana_server_host</name>
- <value>0.0.0.0</value>
- <description>Host name or IP address that Kibana should bind to.</description>
- </property>
- <property require-input="true">
- <name>kibana_log_dir</name>
- <value>/var/log/kibana</value>
- <description>Log directory for Kibana</description>
- </property>
- <property require-input="true">
- <name>kibana_pid_dir</name>
- <value>/var/run/kibana</value>
- <description>PID directory for Kibana</description>
- </property>
- <property require-input="true">
- <name>kibana_es_url</name>
- <value></value>
- <description>The Elasticsearch instance to use for all your queries. (http://eshost:9200)</description>
- </property>
- <property require-input="true">
- <name>kibana_server_port</name>
- <value>5000</value>
- <description>Kibana back end server port to use.</description>
- </property>
- <property require-input="true">
- <name>kibana_default_application</name>
- <value>default</value>
- <description>The default application to load.</description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml
deleted file mode 100755
index d8d0513..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/configuration/kibana-site.xml
+++ /dev/null
@@ -1,113 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
- <!-- kibana.yml -->
- <property>
- <name>content</name>
- <display-name>kibana.yml template</display-name>
- <description>This is the jinja template for kibana.yml file</description>
- <value>
-# Kibana is served by a back end server. This controls which port to use.
-server.port: {{ kibana_port }}
-
-# The host to bind the server to.
-# Kibana (like Elasticsearch) now binds to localhost for security purposes instead of 0.0.0.0 (all addresses). Previous binding to 0.0.0.0 also caused issues for Windows users.
-server.host: {{ kibana_server_host }}
-
-# If you are running kibana behind a proxy, and want to mount it at a path,
-# specify that path here. The basePath can't end in a slash.
-# server.basePath: ""
-
-# The maximum payload size in bytes on incoming server requests.
-# server.maxPayloadBytes: 1048576
-
-# The Elasticsearch instance to use for all your queries.
-elasticsearch.url: {{ es_url }}
-
-# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
-# then the host you use to connect to *this* Kibana instance will be sent.
-# elasticsearch.preserveHost: true
-
-# Kibana uses an index in Elasticsearch to store saved searches, visualizations
-# and dashboards. It will create a new index if it doesn't already exist.
-# kibana.index: ".kibana"
-
-# The default application to load.
-kibana.defaultAppId: "{{ kibana_default_application }}"
-
-# If your Elasticsearch is protected with basic auth, these are the user credentials
-# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana
-# users will still need to authenticate with Elasticsearch (which is proxied through
-# the Kibana server)
-# elasticsearch.username: "user"
-# elasticsearch.password: "pass"
-
-# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted)
-# server.ssl.cert: /path/to/your/server.crt
-# server.ssl.key: /path/to/your/server.key
-
-# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted)
-# elasticsearch.ssl.cert: /path/to/your/client.crt
-# elasticsearch.ssl.key: /path/to/your/client.key
-
-# If you need to provide a CA certificate for your Elasticsearch instance, put
-# the path of the pem file here.
-# elasticsearch.ssl.ca: /path/to/your/CA.pem
-
-# Set to false to have a complete disregard for the validity of the SSL
-# certificate.
-# elasticsearch.ssl.verify: true
-
-# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to
-# request_timeout setting
-# elasticsearch.pingTimeout: 1500
-
-# Time in milliseconds to wait for responses from the back end or elasticsearch.
-# This must be > 0
-# elasticsearch.requestTimeout: 30000
-
-# Time in milliseconds for Elasticsearch to wait for responses from shards.
-# Set to 0 to disable.
-# elasticsearch.shardTimeout: 0
-
-# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying
-# elasticsearch.startupTimeout: 5000
-
-# Set the path to where you would like the process id file to be created.
-# pid.file: /var/run/kibana.pid
-
-# If you would like to send the log output to a file you can set the path below.
-logging.dest: {{ log_dir }}/kibana.log
-
-# Set this to true to suppress all logging output.
-# logging.silent: false
-
-# Set this to true to suppress all logging output except for error messages.
-# logging.quiet: false
-
-# Set this to true to log all events, including system usage information and all requests.
-# logging.verbose: false
- </value>
- <value-attributes>
- <type>content</type>
- </value-attributes>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
deleted file mode 100755
index 06b61a1..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
+++ /dev/null
@@ -1,94 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>KIBANA</name>
- <displayName>Kibana</displayName>
- <comment>Kibana Dashboard</comment>
- <version>5.6.2</version>
- <components>
- <component>
- <name>KIBANA_MASTER</name>
- <displayName>Kibana Server</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <commandScript>
- <script>scripts/kibana_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>LOAD_TEMPLATE</name>
- <background>false</background>
- <commandScript>
- <script>scripts/kibana_master.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </customCommand>
- </customCommands>
- </component>
- </components>
- <osSpecifics>
- <osSpecific>
- <osFamily>redhat6</osFamily>
- <packages>
- <package>
- <name>python-elasticsearch</name>
- </package>
- <package>
- <name>kibana-5.6.2</name>
- </package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>redhat7</osFamily>
- <packages>
- <package>
- <name>python-elasticsearch</name>
- </package>
- <package>
- <name>kibana-5.6.2</name>
- </package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>ubuntu14</osFamily>
- <packages>
- <package>
- <name>kibana=5.6.2</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
- <configuration-dependencies>
- <config-type>kibana-env</config-type>
- <config-type>kibana-site</config-type>
- </configuration-dependencies>
- <restartRequiredAfterChange>true</restartRequiredAfterChange>
- <quickLinksConfigurations>
- <quickLinksConfiguration>
- <fileName>quicklinks.json</fileName>
- <default>true</default>
- </quickLinksConfiguration>
- </quickLinksConfigurations>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
deleted file mode 100644
index 37100cd..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
+++ /dev/null
@@ -1,56 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.logger import Logger
-from resource_management.libraries.functions.get_user_call_output import get_user_call_output
-from resource_management.core.exceptions import ExecutionFailed
-from resource_management.core.exceptions import ComponentIsNotRunning
-
-def service_check(cmd, user, label):
- """
- Executes a service check command that adheres to LSB-compliant
- return codes. The return codes are interpreted as defined
- by the LSB.
-
- See http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
- for more information.
-
- :param cmd: The service check command to execute.
- :param label: The name of the service.
- """
- Logger.info("Performing service check; cmd={0}, user={1}, label={2}".format(cmd, user, label))
- rc, out, err = get_user_call_output(cmd, user, is_checked_call=False)
-
- if len(err) > 0:
- Logger.error(err)
-
- if rc in [1, 2, 3]:
- # if return code in [1, 2, 3], then 'program is not running' or 'program is dead'
- Logger.info("{0} is not running".format(label))
- raise ComponentIsNotRunning()
-
- elif rc == 0:
- # if return code = 0, then 'program is running or service is OK'
- Logger.info("{0} is running".format(label))
-
- else:
- # else service state is unknown
- err_msg = "{0} service check failed; cmd '{1}' returned {2}".format(label, cmd, rc)
- Logger.error(err_msg)
- raise ExecutionFailed(err_msg, rc, out, err)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/__init__.py
deleted file mode 100755
index 8d2bad8..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
\ No newline at end of file