You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/07/14 10:41:42 UTC
[3/5] ambari git commit: AMBARI-21471. ATS going down due to missing
org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin (smohanty)
AMBARI-21471. ATS going down due to missing org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/853a5d4a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/853a5d4a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/853a5d4a
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 853a5d4a2eda1afb5ee4578cf99d0757abc5f95d
Parents: eb1adcb
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Jul 13 22:35:28 2017 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Jul 13 22:38:40 2017 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.5/services/stack_advisor.py | 5 +-
.../src/main/resources/stacks/stack_advisor.py | 19 ++-
.../stacks/2.5/common/test_stack_advisor.py | 150 +++++++++++--------
3 files changed, 105 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/853a5d4a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 3337e8e..4ca74ee 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -19,6 +19,7 @@ limitations under the License.
import math
+
from ambari_commons.str_utils import string_set_equals
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.get_bare_principal import get_bare_principal
@@ -774,9 +775,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME)
putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services)
- stack_root = "/usr/hdp"
- if cluster_env and "stack_root" in cluster_env:
- stack_root = cluster_env["stack_root"]
+ stack_root = self.getStackRoot(services)
timeline_plugin_classes_values = []
timeline_plugin_classpath_values = []
http://git-wip-us.apache.org/repos/asf/ambari/blob/853a5d4a/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 8e08d82..67f7fe0 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -25,6 +25,7 @@ import re
import socket
import string
import traceback
+import json
import sys
import logging
from math import ceil, floor
@@ -34,7 +35,6 @@ from urlparse import urlparse
from resource_management.libraries.functions.data_structure_utils import get_from_dict
from resource_management.core.exceptions import Fail
-
class StackAdvisor(object):
"""
Abstract class implemented by all stack advisors. Stack advisors advise on stack specific questions.
@@ -2006,6 +2006,23 @@ class DefaultStackAdvisor(StackAdvisor):
return mount_points
+ def getStackRoot(self, services):
+ """
+ Gets the stack root associated with the stack
+ :param services: the services structure containing the current configurations
+ :return: the stack root as specified in the config or /usr/hdp
+ """
+ cluster_env = self.getServicesSiteProperties(services, "cluster-env")
+ stack_root = "/usr/hdp"
+ if cluster_env and "stack_root" in cluster_env:
+ stack_root_as_str = cluster_env["stack_root"]
+ stack_roots = json.loads(stack_root_as_str)
+ stack_name = cluster_env["stack_name"]
+ if stack_name in stack_roots:
+ stack_root = stack_roots[stack_name]
+
+ return stack_root
+
def isSecurityEnabled(self, services):
"""
Determines if security is enabled by testing the value of cluster-env/security enabled.
http://git-wip-us.apache.org/repos/asf/ambari/blob/853a5d4a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 50f527d..bf0cbec 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -825,70 +825,80 @@ class TestHDP25StackAdvisor(TestCase):
services = {
"services": [{
- "StackServices": {
- "service_name": "YARN",
- },
- "Versions": {
- "stack_version": "2.5"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "NODEMANAGER",
- "hostnames": ["c6401.ambari.apache.org"]
- }
- }
- ]
- }, {
- "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE",
- "StackServices": {
- "service_name": "HIVE",
- "service_version": "1.2.1.2.5",
- "stack_name": "HDP",
- "stack_version": "2.5"
- },
- "components": [
- {
- "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE",
- "StackServiceComponents": {
- "advertise_version": "true",
- "bulk_commands_display_name": "",
- "bulk_commands_master_component_name": "",
- "cardinality": "0-1",
- "component_category": "MASTER",
- "component_name": "HIVE_SERVER_INTERACTIVE",
- "custom_commands": ["RESTART_LLAP"],
- "decommission_allowed": "false",
- "display_name": "HiveServer2 Interactive",
- "has_bulk_commands_definition": "false",
- "is_client": "false",
- "is_master": "true",
- "reassign_allowed": "false",
- "recovery_enabled": "false",
- "service_name": "HIVE",
- "stack_name": "HDP",
- "stack_version": "2.5",
- "hostnames": ["c6401.ambari.apache.org"]
- },
- "dependencies": []
- },
- {
- "StackServiceComponents": {
- "advertise_version": "true",
- "cardinality": "1+",
- "component_category": "SLAVE",
- "component_name": "NODEMANAGER",
- "display_name": "NodeManager",
- "is_client": "false",
- "is_master": "false",
- "hostnames": [
- "c6403.ambari.apache.org"
- ]
- },
- "dependencies": []
- },
- ]
- }
+ "StackServices": {
+ "service_name": "TEZ"
+ }
+ },
+ {
+ "StackServices": {
+ "service_name": "SPARK"
+ }
+ },
+ {
+ "StackServices": {
+ "service_name": "YARN",
+ },
+ "Versions": {
+ "stack_version": "2.5"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "NODEMANAGER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ }
+ }
+ ]
+ }, {
+ "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE",
+ "StackServices": {
+ "service_name": "HIVE",
+ "service_version": "1.2.1.2.5",
+ "stack_name": "HDP",
+ "stack_version": "2.5"
+ },
+ "components": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE",
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "bulk_commands_display_name": "",
+ "bulk_commands_master_component_name": "",
+ "cardinality": "0-1",
+ "component_category": "MASTER",
+ "component_name": "HIVE_SERVER_INTERACTIVE",
+ "custom_commands": ["RESTART_LLAP"],
+ "decommission_allowed": "false",
+ "display_name": "HiveServer2 Interactive",
+ "has_bulk_commands_definition": "false",
+ "is_client": "false",
+ "is_master": "true",
+ "reassign_allowed": "false",
+ "recovery_enabled": "false",
+ "service_name": "HIVE",
+ "stack_name": "HDP",
+ "stack_version": "2.5",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "hostnames": [
+ "c6403.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ },
+ ]
+ }
],
"changed-configurations": [
{
@@ -898,6 +908,12 @@ class TestHDP25StackAdvisor(TestCase):
}
],
"configurations": {
+ "cluster-env": {
+ "properties": {
+ "stack_root": "{\"HDP\":\"/usr/hdp\"}",
+ "stack_name": "HDP"
+ },
+ },
"capacity-scheduler": {
"properties": {
"capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=60\n'
@@ -960,7 +976,8 @@ class TestHDP25StackAdvisor(TestCase):
"tez.am.resource.memory.mb": "341"
}
}
- }
+ },
+ "ambari-server-properties": {"ambari-server.user":"ambari_user"}
}
clusterData = {
@@ -990,6 +1007,9 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'default')
self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], 'default')
+ self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes'],
+ 'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl,org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin')
+ self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/${hdp.version}/spark/hdpLib/*')
self.assertTrue('hive-interactive-env' not in configurations)
self.assertTrue('property_attributes' not in configurations)