You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2018/04/18 14:59:56 UTC
[26/52] [abbrv] metron git commit: METRON-1462: Separate ES and
Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
new file mode 100755
index 0000000..f0903ac
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
@@ -0,0 +1,95 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from elasticsearch import Elasticsearch
+from elasticsearch.helpers import bulk
+import cPickle as pickle
+import argparse, sys, os.path
+import errno
+import os
+
+
+class DashboardIndex(object):
+
+ def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs):
+ """
+ :arg host: hostname of the node (default: localhost)
+ :arg port: port to use (integer, default: 9200)
+ :arg url_prefix: optional url prefix for elasticsearch
+ :arg timeout: default timeout in seconds (float, default: 10)
+ """
+ self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}])
+
+ def get(self):
+ """
+ Get .kibana index from Elasticsearch
+ """
+ dotkibana = self.es.search(index='.kibana', size = 100)
+ return dotkibana['hits']['hits']
+
+ def load(self,filespec):
+ """
+ Save Index data on local filesystem
+ :args filespec: path/filename for saved file
+ """
+ data=[]
+ with open(filespec,'rb') as fp:
+ data = pickle.load(fp)
+ return data
+
+ def save(self,filename,data):
+ """
+ Save Index data on local filesystem
+ :args filespec: path/filename for saved file
+ """
+ with open(filename,'wb') as fp:
+ pickle.dump(data,fp)
+
+ def put(self,data):
+ """
+ Bulk write data to Elasticsearch
+ :args data: data to be written (note: index name is specified in data)
+ """
+ bulk(self.es,data)
+
+ def main(self,args):
+
+ if args.save:
+ print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec))
+ self.save(filename=args.filespec,data=di.get())
+ else:
+ """
+ Loads Kibana Dashboard definition from disk and replaces .kibana on index
+ :args filespec: path/filename for saved file
+ """
+ if not os.path.isfile(args.filespec):
+ raise IOError(
+ errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
+ self.es.indices.delete(index='.kibana', ignore=[400, 404])
+ self.put(data=di.load(filespec=args.filespec))
+
+if __name__ == '__main__':
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("hostname", help="ES Hostname or IP", type=str)
+ parser.add_argument("port", help="ES Port", type=int)
+ parser.add_argument("filespec", help="file to be pushed from or saved to", type=str)
+ parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true")
+ args = parser.parse_args()
+ di = DashboardIndex(host=args.hostname,port=args.port)
+ di.main(args)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
new file mode 100644
index 0000000..6f38ed5
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
@@ -0,0 +1,233 @@
+{
+ "template" : ".kibana",
+ "mappings" : {
+ "search" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "columns" : {
+ "type" : "keyword"
+ },
+ "description" : {
+ "type" : "text"
+ },
+ "hits" : {
+ "type" : "integer"
+ },
+ "kibanaSavedObjectMeta" : {
+ "properties" : {
+ "searchSourceJSON" : {
+ "type" : "text"
+ }
+ }
+ },
+ "sort" : {
+ "type" : "keyword"
+ },
+ "title" : {
+ "type" : "text"
+ },
+ "version" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "url" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "accessCount" : {
+ "type" : "long"
+ },
+ "accessDate" : {
+ "type" : "date"
+ },
+ "createDate" : {
+ "type" : "date"
+ },
+ "url" : {
+ "type" : "text",
+ "fields" : {
+ "keyword" : {
+ "type" : "keyword",
+ "ignore_above" : 2048
+ }
+ }
+ }
+ }
+ },
+ "dashboard" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "description" : {
+ "type" : "text"
+ },
+ "hits" : {
+ "type" : "integer"
+ },
+ "kibanaSavedObjectMeta" : {
+ "properties" : {
+ "searchSourceJSON" : {
+ "type" : "text"
+ }
+ }
+ },
+ "optionsJSON" : {
+ "type" : "text"
+ },
+ "panelsJSON" : {
+ "type" : "text"
+ },
+ "refreshInterval" : {
+ "properties" : {
+ "display" : {
+ "type" : "keyword"
+ },
+ "pause" : {
+ "type" : "boolean"
+ },
+ "section" : {
+ "type" : "integer"
+ },
+ "value" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "timeFrom" : {
+ "type" : "keyword"
+ },
+ "timeRestore" : {
+ "type" : "boolean"
+ },
+ "timeTo" : {
+ "type" : "keyword"
+ },
+ "title" : {
+ "type" : "text"
+ },
+ "uiStateJSON" : {
+ "type" : "text"
+ },
+ "version" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "index-pattern" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "fieldFormatMap" : {
+ "type" : "text"
+ },
+ "fields" : {
+ "type" : "text"
+ },
+ "intervalName" : {
+ "type" : "keyword"
+ },
+ "notExpandable" : {
+ "type" : "boolean"
+ },
+ "sourceFilters" : {
+ "type" : "text"
+ },
+ "timeFieldName" : {
+ "type" : "keyword"
+ },
+ "title" : {
+ "type" : "text"
+ }
+ }
+ },
+ "timelion-sheet" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "description" : {
+ "type" : "text"
+ },
+ "hits" : {
+ "type" : "integer"
+ },
+ "kibanaSavedObjectMeta" : {
+ "properties" : {
+ "searchSourceJSON" : {
+ "type" : "text"
+ }
+ }
+ },
+ "timelion_chart_height" : {
+ "type" : "integer"
+ },
+ "timelion_columns" : {
+ "type" : "integer"
+ },
+ "timelion_interval" : {
+ "type" : "keyword"
+ },
+ "timelion_other_interval" : {
+ "type" : "keyword"
+ },
+ "timelion_rows" : {
+ "type" : "integer"
+ },
+ "timelion_sheet" : {
+ "type" : "text"
+ },
+ "title" : {
+ "type" : "text"
+ },
+ "version" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "visualization" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "description" : {
+ "type" : "text"
+ },
+ "kibanaSavedObjectMeta" : {
+ "properties" : {
+ "searchSourceJSON" : {
+ "type" : "text"
+ }
+ }
+ },
+ "savedSearchId" : {
+ "type" : "keyword"
+ },
+ "title" : {
+ "type" : "text"
+ },
+ "uiStateJSON" : {
+ "type" : "text"
+ },
+ "version" : {
+ "type" : "integer"
+ },
+ "visState" : {
+ "type" : "text"
+ }
+ }
+ },
+ "server" : {
+ "dynamic" : "strict",
+ "properties" : {
+ "uuid" : {
+ "type" : "keyword"
+ }
+ }
+ },
+ "_default_" : {
+ "dynamic" : "strict"
+ },
+ "config" : {
+ "dynamic" : "true",
+ "properties" : {
+ "buildNum" : {
+ "type" : "keyword"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index ce8c074..1cd6f4c 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -14,8 +14,12 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+import errno
import os
import requests
+
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
from resource_management.core.exceptions import ComponentIsNotRunning
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute
@@ -149,6 +153,34 @@ class Indexing(Script):
cmd.format(params.es_http_url, template_name),
logoutput=True)
+ @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+ def kibana_dashboard_install(self, env):
+ from params import params
+ env.set_params(params)
+
+ Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url))
+
+ kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template')
+ if not os.path.isfile(kibanaTemplate):
+ raise IOError(
+ errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate)
+
+ Logger.info("Loading .kibana index template from %s" % kibanaTemplate)
+ template_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_http_url}/_template/.kibana -d @%s' % kibanaTemplate)
+ Execute(template_cmd, logoutput=True)
+
+ kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json')
+ if not os.path.isfile(kibanaDashboardLoad):
+ raise IOError(
+ errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad)
+
+ Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad)
+
+ kibana_cmd = ambari_format(
+ 'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_http_url}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad)
+ Execute(kibana_cmd, logoutput=True)
+
def zeppelin_notebook_import(self, env):
from params import params
env.set_params(params)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml
index fe76ade..f59324c 100644
--- a/metron-deployment/pom.xml
+++ b/metron-deployment/pom.xml
@@ -49,6 +49,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<modules>
+ <module>packaging/ambari/elasticsearch-mpack</module>
<module>packaging/ambari/metron-mpack</module>
</modules>
</profile>