You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/02/13 22:13:32 UTC
[24/50] [abbrv] ambari git commit: AMBARI-19963 Deviation alerts fail
with "No JSON object could be decoded" (dsen)
AMBARI-19963 Deviation alerts fail with "No JSON object could be decoded" (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fe1704e1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fe1704e1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fe1704e1
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: fe1704e12287685f8ca0fed179eb471556eee37b
Parents: 33caec2
Author: Dmytro Sen <ds...@apache.org>
Authored: Fri Feb 10 19:52:40 2017 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Fri Feb 10 19:52:40 2017 +0200
----------------------------------------------------------------------
.../src/main/python/ambari_commons/network.py | 39 ++++++++++++++++++++
.../package/scripts/metrics_grafana_util.py | 2 +-
.../0.1.0/package/scripts/network.py | 39 --------------------
.../0.1.0/package/scripts/service_check.py | 2 +-
.../package/alerts/alert_metrics_deviation.py | 14 +++++--
.../package/alerts/alert_metrics_deviation.py | 14 +++++--
.../2.0.6/HDFS/test_alert_metrics_deviation.py | 2 +
7 files changed, 65 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe1704e1/ambari-common/src/main/python/ambari_commons/network.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/network.py b/ambari-common/src/main/python/ambari_commons/network.py
new file mode 100644
index 0000000..b5b1cd6
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_commons/network.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import httplib
+import ssl
+
+from resource_management.core.exceptions import Fail
+
+def get_http_connection(host, port, https_enabled=False, ca_certs=None):
+ if https_enabled:
+ if ca_certs:
+ check_ssl_certificate(host, port, ca_certs)
+ return httplib.HTTPSConnection(host, port)
+ else:
+ return httplib.HTTPConnection(host, port)
+
+def check_ssl_certificate(host, port, ca_certs):
+ try:
+ ssl.get_server_certificate((host, port), ssl_version=ssl.PROTOCOL_SSLv23, ca_certs=ca_certs)
+ except (ssl.SSLError) as ssl_error:
+ raise Fail("Failed to verify the SSL certificate for https://{0}:{1} with CA certificate in {2}"
+ .format(host, port, ca_certs))
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe1704e1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
index 84dcd99..a751330 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
@@ -31,7 +31,7 @@ import random
import time
import socket
import ambari_simplejson as json
-import network
+import ambari_commons.network as network
import os
GRAFANA_CONNECT_TRIES = 15
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe1704e1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/network.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/network.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/network.py
deleted file mode 100644
index 672ee53..0000000
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/network.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import httplib
-import ssl
-
-from resource_management.core.exceptions import Fail
-
-def get_http_connection(host, port, https_enabled=False, ca_certs=None):
- if https_enabled:
- if ca_certs:
- check_ssl_certificate(host, port, ca_certs)
- return httplib.HTTPSConnection(host, port)
- else:
- return httplib.HTTPConnection(host, port)
-
-def check_ssl_certificate(host, port, ca_certs):
- try:
- ssl.get_server_certificate((host, port), ssl_version=ssl.PROTOCOL_SSLv23, ca_certs=ca_certs)
- except (ssl.SSLError) as ssl_error:
- raise Fail("Failed to verify the SSL certificate for AMS Collector https://{0}:{1} with CA certificate in {2}"
- .format(host, port, ca_certs))
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe1704e1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
index 2bc4363..e753958 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
@@ -28,7 +28,7 @@ from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons.parallel_processing import PrallelProcessResult, execute_in_parallel, SUCCESS
import httplib
-import network
+import ambari_commons.network as network
import urllib
import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
import os
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe1704e1/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py
index 8a06f56..bc2102a 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py
@@ -24,6 +24,8 @@ import logging
import urllib
import time
import urllib2
+import os
+import ambari_commons.network as network
from resource_management import Environment
from ambari_commons.aggregate_functions import sample_standard_deviation, mean
@@ -55,6 +57,7 @@ SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
SMOKEUSER_KEY = '{{cluster-env/smokeuser}}'
EXECUTABLE_SEARCH_PATHS = '{{kerberos-env/executable_search_paths}}'
+AMS_HTTP_POLICY = '{{ams-site/timeline.metrics.service.http.policy}}'
METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}'
METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}'
METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}'
@@ -105,7 +108,7 @@ def get_tokens():
EXECUTABLE_SEARCH_PATHS, NN_HTTPS_ADDRESS_KEY, SMOKEUSER_KEY,
KERBEROS_KEYTAB, KERBEROS_PRINCIPAL, SECURITY_ENABLED_KEY,
METRICS_COLLECTOR_VIP_HOST_KEY, METRICS_COLLECTOR_VIP_PORT_KEY,
- METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY)
+ METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, AMS_HTTP_POLICY)
def execute(configurations={}, parameters={}, host_name=None):
"""
@@ -310,9 +313,14 @@ def execute(configurations={}, parameters={}, host_name=None):
encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters)
+ ams_monitor_conf_dir = "/etc/ambari-metrics-monitor/conf"
+ metric_truststore_ca_certs='ca.pem'
+ ca_certs = os.path.join(ams_monitor_conf_dir,
+ metric_truststore_ca_certs)
+ metric_collector_https_enabled = str(configurations[AMS_HTTP_POLICY]) == "HTTPS_ONLY"
+
try:
- conn = httplib.HTTPConnection(collector_host, int(collector_port),
- timeout=connection_timeout)
+ conn = network.get_http_connection(collector_host, int(collector_port), metric_collector_https_enabled, ca_certs)
conn.request("GET", AMS_METRICS_GET_URL % encoded_get_metrics_parameters)
response = conn.getresponse()
data = response.read()
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe1704e1/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/alerts/alert_metrics_deviation.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/alerts/alert_metrics_deviation.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/alerts/alert_metrics_deviation.py
index 8a06f56..bc2102a 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/alerts/alert_metrics_deviation.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/alerts/alert_metrics_deviation.py
@@ -24,6 +24,8 @@ import logging
import urllib
import time
import urllib2
+import os
+import ambari_commons.network as network
from resource_management import Environment
from ambari_commons.aggregate_functions import sample_standard_deviation, mean
@@ -55,6 +57,7 @@ SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
SMOKEUSER_KEY = '{{cluster-env/smokeuser}}'
EXECUTABLE_SEARCH_PATHS = '{{kerberos-env/executable_search_paths}}'
+AMS_HTTP_POLICY = '{{ams-site/timeline.metrics.service.http.policy}}'
METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}'
METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}'
METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}'
@@ -105,7 +108,7 @@ def get_tokens():
EXECUTABLE_SEARCH_PATHS, NN_HTTPS_ADDRESS_KEY, SMOKEUSER_KEY,
KERBEROS_KEYTAB, KERBEROS_PRINCIPAL, SECURITY_ENABLED_KEY,
METRICS_COLLECTOR_VIP_HOST_KEY, METRICS_COLLECTOR_VIP_PORT_KEY,
- METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY)
+ METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, AMS_HTTP_POLICY)
def execute(configurations={}, parameters={}, host_name=None):
"""
@@ -310,9 +313,14 @@ def execute(configurations={}, parameters={}, host_name=None):
encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters)
+ ams_monitor_conf_dir = "/etc/ambari-metrics-monitor/conf"
+ metric_truststore_ca_certs='ca.pem'
+ ca_certs = os.path.join(ams_monitor_conf_dir,
+ metric_truststore_ca_certs)
+ metric_collector_https_enabled = str(configurations[AMS_HTTP_POLICY]) == "HTTPS_ONLY"
+
try:
- conn = httplib.HTTPConnection(collector_host, int(collector_port),
- timeout=connection_timeout)
+ conn = network.get_http_connection(collector_host, int(collector_port), metric_collector_https_enabled, ca_certs)
conn.request("GET", AMS_METRICS_GET_URL % encoded_get_metrics_parameters)
response = conn.getresponse()
data = response.read()
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe1704e1/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py
index 1e35e6f..09e8886 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py
@@ -81,6 +81,7 @@ class TestAlertMetricsDeviation(RMFTestCase):
'{{hdfs-site/dfs.namenode.https-address}}': 'c6401.ambari.apache.org:50470',
'{{hdfs-site/dfs.http.policy}}': 'HTTP_ONLY',
'{{ams-site/timeline.metrics.service.webapp.address}}': '0.0.0.0:6188',
+ '{{ams-site/timeline.metrics.service.http.policy}}' : 'HTTP_ONLY',
'{{hdfs-site/dfs.namenode.http-address}}': 'c6401.ambari.apache.org:50070',
'{{cluster-env/security_enabled}}': 'false',
'{{cluster-env/smokeuser}}': 'ambari-qa',
@@ -105,6 +106,7 @@ class TestAlertMetricsDeviation(RMFTestCase):
'{{hdfs-site/dfs.namenode.https-address}}': 'c6401.ambari.apache.org:50470',
'{{hdfs-site/dfs.http.policy}}': 'HTTP_ONLY',
'{{ams-site/timeline.metrics.service.webapp.address}}': '0.0.0.0:6188',
+ '{{ams-site/timeline.metrics.service.http.policy}}' : 'HTTP_ONLY',
'{{hdfs-site/dfs.namenode.http-address}}': 'c6401.ambari.apache.org:50070',
'{{cluster-env/security_enabled}}': 'false',
'{{cluster-env/smokeuser}}': 'ambari-qa',