You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2015/04/24 17:37:14 UTC
ambari git commit: AMBARI-10728. Changes for yarn service check by
ditributed shell jar.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 239d3ac75 -> b034bab2f
AMBARI-10728. Changes for yarn service check by ditributed shell jar.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b034bab2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b034bab2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b034bab2
Branch: refs/heads/trunk
Commit: b034bab2f68e16d3ffe3d184dd38461bfe71cd2c
Parents: 239d3ac
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Tue Apr 21 06:01:47 2015 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Tue Apr 21 06:01:47 2015 +0300
----------------------------------------------------------------------
.../2.1.0.2.0/package/scripts/params_linux.py | 3 ++
.../2.1.0.2.0/package/scripts/service_check.py | 50 ++++++++++++++++----
.../catalog/UpgradeCatalog_2.1_to_2.2.4.json | 2 +-
.../2.0.6/YARN/test_yarn_service_check.py | 43 +++++++++--------
4 files changed, 68 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b034bab2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index da10bbe..96e6e30 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -153,6 +153,9 @@ exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.node
ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
has_ats = not len(ats_host) == 0
+nm_hosts = default("/clusterHostInfo/nm_hosts", [])
+number_of_nm = len(nm_hosts)
+
# default kinit commands
rm_kinit_cmd = ""
yarn_timelineservice_kinit_cmd = ""
http://git-wip-us.apache.org/repos/asf/ambari/blob/b034bab2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
index 7d095b9..b3e5e4d 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
@@ -22,9 +22,13 @@ Ambari Agent
from resource_management.libraries.functions.version import compare_versions
from resource_management import *
import sys
+import json
+import re
+import subprocess
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyImpl
+CURL_CONNECTION_TIMEOUT = '5'
class ServiceCheck(Script):
def service_check(self, env):
@@ -85,7 +89,7 @@ class ServiceCheckDefault(ServiceCheck):
path_to_distributed_shell_jar = "/usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar"
yarn_distrubuted_shell_check_cmd = format("yarn org.apache.hadoop.yarn.applications.distributedshell.Client "
- "-shell_command ls -jar {path_to_distributed_shell_jar}")
+ "-shell_command ls -num_containers {number_of_nm} -jar {path_to_distributed_shell_jar}")
if params.security_enabled:
kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
@@ -93,13 +97,43 @@ class ServiceCheckDefault(ServiceCheck):
else:
smoke_cmd = yarn_distrubuted_shell_check_cmd
- Execute(smoke_cmd,
- tries=3,
- try_sleep=5,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- user=params.smokeuser,
- logoutput=True
- )
+ return_code, out = shell.checked_call(smoke_cmd,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ user=params.smokeuser,
+ )
+
+ m = re.search("appTrackingUrl=(.*),\s", out)
+ app_url = m.group(1)
+
+ splitted_app_url = str(app_url).split('/')
+
+ for item in splitted_app_url:
+ if "application" in item:
+ application_name = item
+
+ json_response_received = False
+ for rm_host in params.rm_hosts:
+ info_app_url = "http://" + rm_host + ":" + params.rm_port + "/ws/v1/cluster/apps/" + application_name
+
+ get_app_info_cmd = "curl --negotiate -u : -sL --connect-timeout " + CURL_CONNECTION_TIMEOUT + " " + info_app_url
+
+ return_code, stdout = shell.checked_call(get_app_info_cmd,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ user=params.smokeuser,
+ )
+
+ try:
+ json_response = json.loads(stdout)
+
+ json_response_received = True
+
+ if json_response['app']['state'] != "FINISHED" or json_response['app']['finalStatus'] != "SUCCEEDED":
+ raise Exception("Application " + app_url + " state/status is not valid. Should be FINISHED/SUCCEEDED.")
+ except Exception as e:
+ pass
+
+ if not json_response_received:
+ raise Exception("Could not get json response from YARN API")
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/ambari/blob/b034bab2/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json
index 3816c48..4eb4ac9 100644
--- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json
@@ -509,4 +509,4 @@
}
}
]
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b034bab2/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py
index 1376fef..65447c6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py
@@ -18,6 +18,7 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
import os
+import re
from mock.mock import MagicMock, call, patch
from stacks.utils.RMFTestCase import *
@@ -27,40 +28,40 @@ class TestServiceCheck(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
STACK_VERSION = "2.0.6"
- def test_service_check_default(self):
+ @patch("re.search")
+ def test_service_check_default(self, re_search_mock):
+ m = MagicMock()
+ re_search_mock.return_value = m
+ m.group.return_value = "http://c6402.ambari.apache.org:8088/proxy/application_1429699682952_0010/"
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py",
classname="ServiceCheck",
command="service_check",
config_file="default.json",
hdp_stack_version = self.STACK_VERSION,
- target = RMFTestCase.TARGET_COMMON_SERVICES
- )
- self.assertResourceCalled('Execute', 'yarn org.apache.hadoop.yarn.applications.distributedshell.Client '
- '-shell_command ls -jar /usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar',
- logoutput = True,
- path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
- tries = 3,
- user = 'ambari-qa',
- try_sleep = 5,
+ target = RMFTestCase.TARGET_COMMON_SERVICES,
+ checked_call_mocks = [(0, "some test text, appTrackingUrl=http:"
+ "//c6402.ambari.apache.org:8088/proxy/application_1429885383763_0001/, some test text"),
+ (0, "{ \"app\": {\"state\": \"FINISHED\",\"finalStatus\": \"SUCCEEDED\"}}")]
)
self.assertNoMoreResources()
- def test_service_check_secured(self):
+
+ @patch("re.search")
+ def test_service_check_secured(self, re_search_mock):
+ m = MagicMock()
+ re_search_mock.return_value = m
+ m.group.return_value = "http://c6402.ambari.apache.org:8088/proxy/application_1429699682952_0010/"
+
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py",
classname="ServiceCheck",
command="service_check",
config_file="secured.json",
hdp_stack_version = self.STACK_VERSION,
- target = RMFTestCase.TARGET_COMMON_SERVICES
- )
- self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab '
- 'ambari-qa@EXAMPLE.COM; yarn org.apache.hadoop.yarn.applications.distributedshell.Client '
- '-shell_command ls -jar /usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar',
- logoutput = True,
- path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
- tries = 3,
- user = 'ambari-qa',
- try_sleep = 5,
+ target = RMFTestCase.TARGET_COMMON_SERVICES,
+ checked_call_mocks = [(0, "some test text, appTrackingUrl=http:"
+ "//c6402.ambari.apache.org:8088/proxy/application_1429885383763_0001/, some test text"),
+ (0, "{ \"app\": {\"state\": \"FINISHED\",\"finalStatus\": \"SUCCEEDED\"}}")]
)
+
self.assertNoMoreResources()