You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/02/10 17:13:16 UTC
[19/24] ambari git commit: AMBARI-14962 : Auto configure AMS UI to
use AMS Datasource (avijayan)
AMBARI-14962 : Auto configure AMS UI to use AMS Datasource (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/14396f2b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/14396f2b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/14396f2b
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 14396f2b74115744d6f4c93b97a9911cd1e58ff1
Parents: 38a5225
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Mon Feb 8 13:22:15 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Tue Feb 9 08:18:51 2016 -0800
----------------------------------------------------------------------
.../0.1.0/package/scripts/metrics_grafana.py | 2 +
.../package/scripts/metrics_grafana_util.py | 131 +++++++++++++++++++
.../metrics_grafana_datasource.json.j2 | 15 +++
.../AMBARI_METRICS/test_metrics_grafana.py | 19 ++-
4 files changed, 163 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/14396f2b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py
index f876036..d96309c 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py
@@ -22,6 +22,7 @@ from resource_management import Script, Execute
from resource_management.libraries.functions import format
from status import check_service_status
from ams import ams
+from metrics_grafana_util import create_ams_datasource
class AmsGrafana(Script):
def install(self, env):
@@ -46,6 +47,7 @@ class AmsGrafana(Script):
Execute(start_cmd,
user=params.ams_user
)
+ create_ams_datasource()
def stop(self, env):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/14396f2b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
new file mode 100644
index 0000000..c4a91e1
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.core.logger import Logger
+from resource_management.core.base import Fail
+from resource_management import Template
+
+import httplib
+import time
+import socket
+import json
+
+def create_ams_datasource():
+
+ import params
+ GRAFANA_CONNECT_TRIES = 5
+ GRAFANA_CONNECT_TIMEOUT = 15
+ GRAFANA_URL = "/api/datasources"
+ METRICS_GRAFANA_DATASOURCE_NAME = "AMBARI_METRICS"
+
+ headers = {"Content-type": "application/json"}
+
+ Logger.info("Checking if AMS Grafana datasource already exists")
+ Logger.info("Connecting (GET) to %s:%s%s" % (params.hostname,
+ params.ams_grafana_port,
+ GRAFANA_URL))
+
+ conn = httplib.HTTPConnection(params.hostname,
+ int(params.ams_grafana_port))
+
+ conn.request("GET", GRAFANA_URL)
+ response = conn.getresponse()
+ Logger.info("Http response: %s %s" % (response.status, response.reason))
+
+ if(response.status == 200):
+ datasources = response.read()
+ datasources_json = json.loads(datasources)
+ for i in xrange(0, len(datasources_json)):
+ datasource_name = datasources_json[i]["name"]
+ if(datasource_name == METRICS_GRAFANA_DATASOURCE_NAME):
+
+ Logger.info("Ambari Metrics Grafana datasource already present. Checking Metrics Collector URL")
+ datasource_url = datasources_json[i]["url"]
+
+ if datasource_url == (params.metric_collector_host + ":" + params.metric_collector_port
+ + "/ws/v1/timeline/metrics") :
+ Logger.info("Metrics Collector URL validation succeeded. Skipping datasource creation")
+ GRAFANA_CONNECT_TRIES = 0 # No need to create datasource again
+
+ else: # Metrics datasource present, but collector host is wrong.
+
+ Logger.info("Metrics Collector URL validation failed.")
+ datasource_id = datasources_json[i]["id"]
+ Logger.info("Deleting obselete Metrics datasource.")
+ conn = httplib.HTTPConnection(params.hostname, int(params.ams_grafana_port))
+ conn.request("DELETE", GRAFANA_URL + "/" + str(datasource_id))
+ response = conn.getresponse()
+ Logger.info("Http response: %s %s" % (response.status, response.reason))
+
+ break
+ else:
+ Logger.info("Error checking for Ambari Metrics Grafana datasource. Will attempt to create.")
+
+ if GRAFANA_CONNECT_TRIES > 0:
+ Logger.info("Attempting to create Ambari Metrics Grafana datasource")
+
+ for i in xrange(0, GRAFANA_CONNECT_TRIES):
+ try:
+ ams_datasource_json = Template('metrics_grafana_datasource.json.j2',
+ ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME,
+ ams_collector_host=params.metric_collector_host,
+ ams_collector_port=params.metric_collector_port).get_content()
+
+ Logger.info("Generated datasource:\n%s" % ams_datasource_json)
+
+ Logger.info("Connecting (POST) to %s:%s%s" % (params.hostname,
+ params.ams_grafana_port,
+ GRAFANA_URL))
+ conn = httplib.HTTPConnection(params.hostname,
+ int(params.ams_grafana_port))
+ conn.request("POST", GRAFANA_URL, ams_datasource_json, headers)
+
+ response = conn.getresponse()
+ Logger.info("Http response: %s %s" % (response.status, response.reason))
+ except (httplib.HTTPException, socket.error) as ex:
+ if i < GRAFANA_CONNECT_TRIES - 1:
+ time.sleep(GRAFANA_CONNECT_TIMEOUT)
+ Logger.info("Connection to Grafana failed. Next retry in %s seconds."
+ % (GRAFANA_CONNECT_TIMEOUT))
+ continue
+ else:
+ raise Fail("Ambari Metrics Grafana datasource not created")
+
+ data = response.read()
+ Logger.info("Http data: %s" % data)
+ conn.close()
+
+ if response.status == 200:
+ Logger.info("Ambari Metrics Grafana data source created.")
+ break
+ elif response.status == 500:
+ Logger.info("Ambari Metrics Grafana data source creation failed. Not retrying.")
+ raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" %
+ (response.status, response.reason, data))
+ else:
+ Logger.info("Ambari Metrics Grafana data source creation failed.")
+ if i < GRAFANA_CONNECT_TRIES - 1:
+ time.sleep(GRAFANA_CONNECT_TIMEOUT)
+ Logger.info("Next retry in %s seconds."
+ % (GRAFANA_CONNECT_TIMEOUT))
+ else:
+ raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" %
+ (response.status, response.reason, data))
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/14396f2b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2
new file mode 100644
index 0000000..3edc6fb
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2
@@ -0,0 +1,15 @@
+{
+ "name": "{{ams_datasource_name}}",
+ "type": "ambarimetrics",
+ "access": "proxy",
+ "url": "{{ams_collector_host}}:{{ams_collector_port}}/ws/v1/timeline/metrics",
+ "password": "",
+ "user": "",
+ "database": "",
+ "basicAuth": false,
+ "basicAuthUser": "",
+ "basicAuthPassword": "",
+ "withCredentials": false,
+ "isDefault": false,
+ "jsonData": {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/14396f2b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py
index 5bc1412..755bb4f 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py
@@ -18,17 +18,27 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
-from mock.mock import MagicMock, patch
+from mock.mock import MagicMock, patch, call
from stacks.utils.RMFTestCase import *
+import os, sys
@patch("os.path.exists", new = MagicMock(return_value=True))
@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
class TestMetricsGrafana(RMFTestCase):
- COMMON_SERVICES_PACKAGE_DIR = "AMBARI_METRICS/0.1.0/package"
+ COMMON_SERVICES_PACKAGE_DIR = "AMBARI_METRICS/0.1.0/package/scripts"
STACK_VERSION = "2.0.6"
- def test_start(self):
- self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metrics_grafana.py",
+ file_path = os.path.dirname(os.path.abspath(__file__))
+ file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path)))))
+ file_path = os.path.join(file_path, "main", "resources", "common-services", COMMON_SERVICES_PACKAGE_DIR)
+
+ sys.path.append(file_path)
+ global metrics_grafana_util
+ import metrics_grafana_util
+
+ @patch("metrics_grafana_util.create_ams_datasource")
+ def test_start(self, create_ams_datasource_mock):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/metrics_grafana.py",
classname = "AmsGrafana",
command = "start",
config_file="default.json",
@@ -43,6 +53,7 @@ class TestMetricsGrafana(RMFTestCase):
self.assertResourceCalled('Execute', '/usr/sbin/ambari-metrics-grafana start',
user = 'ams'
)
+ create_ams_datasource_mock.assertCalled()
self.assertNoMoreResources()
def assert_configure(self):