You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/04/05 15:08:03 UTC
[3/7] metron git commit: METRON-1462: Separate ES and Kibana from
Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
deleted file mode 100755
index f0903ac..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from elasticsearch import Elasticsearch
-from elasticsearch.helpers import bulk
-import cPickle as pickle
-import argparse, sys, os.path
-import errno
-import os
-
-
-class DashboardIndex(object):
-
- def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs):
- """
- :arg host: hostname of the node (default: localhost)
- :arg port: port to use (integer, default: 9200)
- :arg url_prefix: optional url prefix for elasticsearch
- :arg timeout: default timeout in seconds (float, default: 10)
- """
- self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}])
-
- def get(self):
- """
- Get .kibana index from Elasticsearch
- """
- dotkibana = self.es.search(index='.kibana', size = 100)
- return dotkibana['hits']['hits']
-
- def load(self,filespec):
- """
- Save Index data on local filesystem
- :args filespec: path/filename for saved file
- """
- data=[]
- with open(filespec,'rb') as fp:
- data = pickle.load(fp)
- return data
-
- def save(self,filename,data):
- """
- Save Index data on local filesystem
- :args filespec: path/filename for saved file
- """
- with open(filename,'wb') as fp:
- pickle.dump(data,fp)
-
- def put(self,data):
- """
- Bulk write data to Elasticsearch
- :args data: data to be written (note: index name is specified in data)
- """
- bulk(self.es,data)
-
- def main(self,args):
-
- if args.save:
- print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec))
- self.save(filename=args.filespec,data=di.get())
- else:
- """
- Loads Kibana Dashboard definition from disk and replaces .kibana on index
- :args filespec: path/filename for saved file
- """
- if not os.path.isfile(args.filespec):
- raise IOError(
- errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
- self.es.indices.delete(index='.kibana', ignore=[400, 404])
- self.put(data=di.load(filespec=args.filespec))
-
-if __name__ == '__main__':
-
- parser = argparse.ArgumentParser()
- parser.add_argument("hostname", help="ES Hostname or IP", type=str)
- parser.add_argument("port", help="ES Port", type=int)
- parser.add_argument("filespec", help="file to be pushed from or saved to", type=str)
- parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true")
- args = parser.parse_args()
- di = DashboardIndex(host=args.hostname,port=args.port)
- di.main(args)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
deleted file mode 100644
index 6f38ed5..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
+++ /dev/null
@@ -1,233 +0,0 @@
-{
- "template" : ".kibana",
- "mappings" : {
- "search" : {
- "dynamic" : "strict",
- "properties" : {
- "columns" : {
- "type" : "keyword"
- },
- "description" : {
- "type" : "text"
- },
- "hits" : {
- "type" : "integer"
- },
- "kibanaSavedObjectMeta" : {
- "properties" : {
- "searchSourceJSON" : {
- "type" : "text"
- }
- }
- },
- "sort" : {
- "type" : "keyword"
- },
- "title" : {
- "type" : "text"
- },
- "version" : {
- "type" : "integer"
- }
- }
- },
- "url" : {
- "dynamic" : "strict",
- "properties" : {
- "accessCount" : {
- "type" : "long"
- },
- "accessDate" : {
- "type" : "date"
- },
- "createDate" : {
- "type" : "date"
- },
- "url" : {
- "type" : "text",
- "fields" : {
- "keyword" : {
- "type" : "keyword",
- "ignore_above" : 2048
- }
- }
- }
- }
- },
- "dashboard" : {
- "dynamic" : "strict",
- "properties" : {
- "description" : {
- "type" : "text"
- },
- "hits" : {
- "type" : "integer"
- },
- "kibanaSavedObjectMeta" : {
- "properties" : {
- "searchSourceJSON" : {
- "type" : "text"
- }
- }
- },
- "optionsJSON" : {
- "type" : "text"
- },
- "panelsJSON" : {
- "type" : "text"
- },
- "refreshInterval" : {
- "properties" : {
- "display" : {
- "type" : "keyword"
- },
- "pause" : {
- "type" : "boolean"
- },
- "section" : {
- "type" : "integer"
- },
- "value" : {
- "type" : "integer"
- }
- }
- },
- "timeFrom" : {
- "type" : "keyword"
- },
- "timeRestore" : {
- "type" : "boolean"
- },
- "timeTo" : {
- "type" : "keyword"
- },
- "title" : {
- "type" : "text"
- },
- "uiStateJSON" : {
- "type" : "text"
- },
- "version" : {
- "type" : "integer"
- }
- }
- },
- "index-pattern" : {
- "dynamic" : "strict",
- "properties" : {
- "fieldFormatMap" : {
- "type" : "text"
- },
- "fields" : {
- "type" : "text"
- },
- "intervalName" : {
- "type" : "keyword"
- },
- "notExpandable" : {
- "type" : "boolean"
- },
- "sourceFilters" : {
- "type" : "text"
- },
- "timeFieldName" : {
- "type" : "keyword"
- },
- "title" : {
- "type" : "text"
- }
- }
- },
- "timelion-sheet" : {
- "dynamic" : "strict",
- "properties" : {
- "description" : {
- "type" : "text"
- },
- "hits" : {
- "type" : "integer"
- },
- "kibanaSavedObjectMeta" : {
- "properties" : {
- "searchSourceJSON" : {
- "type" : "text"
- }
- }
- },
- "timelion_chart_height" : {
- "type" : "integer"
- },
- "timelion_columns" : {
- "type" : "integer"
- },
- "timelion_interval" : {
- "type" : "keyword"
- },
- "timelion_other_interval" : {
- "type" : "keyword"
- },
- "timelion_rows" : {
- "type" : "integer"
- },
- "timelion_sheet" : {
- "type" : "text"
- },
- "title" : {
- "type" : "text"
- },
- "version" : {
- "type" : "integer"
- }
- }
- },
- "visualization" : {
- "dynamic" : "strict",
- "properties" : {
- "description" : {
- "type" : "text"
- },
- "kibanaSavedObjectMeta" : {
- "properties" : {
- "searchSourceJSON" : {
- "type" : "text"
- }
- }
- },
- "savedSearchId" : {
- "type" : "keyword"
- },
- "title" : {
- "type" : "text"
- },
- "uiStateJSON" : {
- "type" : "text"
- },
- "version" : {
- "type" : "integer"
- },
- "visState" : {
- "type" : "text"
- }
- }
- },
- "server" : {
- "dynamic" : "strict",
- "properties" : {
- "uuid" : {
- "type" : "keyword"
- }
- }
- },
- "_default_" : {
- "dynamic" : "strict"
- },
- "config" : {
- "dynamic" : "true",
- "properties" : {
- "buildNum" : {
- "type" : "keyword"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
deleted file mode 100755
index a15f709..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
+++ /dev/null
@@ -1,119 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import errno
-import os
-
-from ambari_commons.os_check import OSCheck
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import Execute
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-from resource_management.libraries.functions.format import format as ambari_format
-from resource_management.libraries.script import Script
-
-from common import service_check
-
-class Kibana(Script):
-
- def install(self, env):
- import params
- env.set_params(params)
- Logger.info("Installing Kibana")
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- import params
- env.set_params(params)
- Logger.info("Configuring Kibana")
-
- directories = [params.log_dir, params.pid_dir, params.conf_dir]
- Directory(directories,
- mode=0755,
- owner=params.kibana_user,
- group=params.kibana_user
- )
-
- File("{0}/kibana.yml".format(params.conf_dir),
- owner=params.kibana_user,
- content=InlineTemplate(params.kibana_yml_template)
- )
-
- def stop(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- Logger.info("Stopping Kibana")
- Execute("service kibana stop")
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- self.configure(env)
- Logger.info("Starting Kibana")
- Execute("service kibana start")
-
- def restart(self, env):
- import params
- env.set_params(params)
- self.configure(env)
- Logger.info("Restarting Kibana")
- Execute("service kibana restart")
-
- def status(self, env):
- import params
- env.set_params(params)
- Logger.info('Status check Kibana')
- service_check("service kibana status", user=params.kibana_user, label="Kibana")
-
- @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
- def load_template(self, env):
- import params
- env.set_params(params)
-
- hostname = ambari_format("{es_host}")
- port = int(ambari_format("{es_port}"))
-
- Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname, port))
-
- kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template')
- if not os.path.isfile(kibanaTemplate):
- raise IOError(
- errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate)
-
- Logger.info("Loading .kibana index template from %s" % kibanaTemplate)
- template_cmd = ambari_format(
- 'curl -s -XPOST http://{es_host}:{es_port}/_template/.kibana -d @%s' % kibanaTemplate)
- Execute(template_cmd, logoutput=True)
-
- kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json')
- if not os.path.isfile(kibanaDashboardLoad):
- raise IOError(
- errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad)
-
- Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad)
-
- kibana_cmd = ambari_format(
- 'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_host}:{es_port}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad)
- Execute(kibana_cmd, logoutput=True)
-
-
-if __name__ == "__main__":
- Kibana().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
deleted file mode 100755
index ef4cb62..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Kibana Params configurations
-
-"""
-
-from urlparse import urlparse
-
-from resource_management.libraries.functions import format
-from resource_management.libraries.script import Script
-
-# server configurations
-config = Script.get_config()
-
-kibana_home = '/usr/share/kibana/'
-kibana_bin = '/usr/share/kibana/bin/'
-
-conf_dir = "/etc/kibana"
-kibana_user = config['configurations']['kibana-env']['kibana_user']
-kibana_group = config['configurations']['kibana-env']['kibana_group']
-log_dir = config['configurations']['kibana-env']['kibana_log_dir']
-pid_dir = config['configurations']['kibana-env']['kibana_pid_dir']
-pid_file = format("{pid_dir}/kibanasearch.pid")
-es_url = config['configurations']['kibana-env']['kibana_es_url']
-parsed = urlparse(es_url)
-es_host = parsed.netloc.split(':')[0]
-es_port = parsed.netloc.split(':')[1]
-kibana_port = config['configurations']['kibana-env']['kibana_server_port']
-kibana_server_host = config['configurations']['kibana-env']['kibana_server_host']
-kibana_default_application = config['configurations']['kibana-env']['kibana_default_application']
-hostname = config['hostname']
-java64_home = config['hostLevelParams']['java_home']
-kibana_yml_template = config['configurations']['kibana-site']['content']
-
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
deleted file mode 100755
index 448e102..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "metron_ui",
- "label": "Metron UI",
- "requires_user_name": "false",
- "component_name": "KIBANA_MASTER",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "kibana_server_port",
- "http_default_port": "5601",
- "https_property": "kibana_server_port",
- "https_default_port": "5601",
- "regex": "^(\\d+)$",
- "site": "kibana-env"
- }
- }
- ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 6a20af8..38bd94a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -211,6 +211,15 @@
</commandScript>
</customCommand>
<customCommand>
+ <name>KIBANA_DASHBOARD_INSTALL</name>
+ <background>false</background>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ <customCommand>
<name>ZEPPELIN_NOTEBOOK_IMPORT</name>
<commandScript>
<script>scripts/indexing_master.py</script>
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py
new file mode 100755
index 0000000..8d2bad8
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file