You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/07/18 15:03:00 UTC
[24/50] [abbrv] ambari git commit: AMBARI-21471. ATS going down due
to missing org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin
(smohanty)
AMBARI-21471. ATS going down due to missing org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/afea7bb7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/afea7bb7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/afea7bb7
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: afea7bb72475fa3d6938aa049dd2db99f3adc133
Parents: d2c6d53
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Jul 13 22:35:28 2017 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Jul 13 22:35:28 2017 -0700
----------------------------------------------------------------------
.../4.0/stack-advisor/stack_advisor_25.py | 4 +-
.../stacks/HDP/2.5/services/stack_advisor.py | 5 +-
.../src/main/resources/stacks/stack_advisor.py | 18 +++
.../stacks/2.5/common/test_stack_advisor.py | 150 +++++++++++--------
4 files changed, 106 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/afea7bb7/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py
index 1f0ae18..eb7d370 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py
@@ -734,9 +734,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME)
putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services)
- stack_root = "/usr/hdp"
- if "cluster-env" in services["configurations"] and "stack_root" in services["configurations"]["cluster-env"]["properties"]:
- stack_root = services["configurations"]["cluster-env"]["properties"]["stack_root"]
+ stack_root = self.getStackRoot(services)
timeline_plugin_classes_values = []
timeline_plugin_classpath_values = []
http://git-wip-us.apache.org/repos/asf/ambari/blob/afea7bb7/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 638e79a..97c49f3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -19,6 +19,7 @@ limitations under the License.
import math
+
from ambari_commons.str_utils import string_set_equals
from resource_management.core.logger import Logger
from resource_management.core.exceptions import Fail
@@ -775,9 +776,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME)
putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services)
- stack_root = "/usr/hdp"
- if cluster_env and "stack_root" in cluster_env:
- stack_root = cluster_env["stack_root"]
+ stack_root = self.getStackRoot(services)
timeline_plugin_classes_values = []
timeline_plugin_classpath_values = []
http://git-wip-us.apache.org/repos/asf/ambari/blob/afea7bb7/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index e393b9c..1e0d83a 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -22,6 +22,7 @@ import os
import re
import socket
import traceback
+import json
class StackAdvisor(object):
"""
@@ -1144,6 +1145,23 @@ class DefaultStackAdvisor(StackAdvisor):
return mount_points
+ def getStackRoot(self, services):
+ """
+ Gets the stack root associated with the stack
+ :param services: the services structure containing the current configurations
+ :return: the stack root as specified in the config or /usr/hdp
+ """
+ cluster_env = self.getServicesSiteProperties(services, "cluster-env")
+ stack_root = "/usr/hdp"
+ if cluster_env and "stack_root" in cluster_env:
+ stack_root_as_str = cluster_env["stack_root"]
+ stack_roots = json.loads(stack_root_as_str)
+ stack_name = cluster_env["stack_name"]
+ if stack_name in stack_roots:
+ stack_root = stack_roots[stack_name]
+
+ return stack_root
+
def isSecurityEnabled(self, services):
"""
Determines if security is enabled by testing the value of cluster-env/security enabled.
http://git-wip-us.apache.org/repos/asf/ambari/blob/afea7bb7/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 7bc9272..222f100 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -825,70 +825,80 @@ class TestHDP25StackAdvisor(TestCase):
services = {
"services": [{
- "StackServices": {
- "service_name": "YARN",
- },
- "Versions": {
- "stack_version": "2.5"
- },
- "components": [
- {
- "StackServiceComponents": {
- "component_name": "NODEMANAGER",
- "hostnames": ["c6401.ambari.apache.org"]
- }
- }
- ]
- }, {
- "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE",
- "StackServices": {
- "service_name": "HIVE",
- "service_version": "1.2.1.2.5",
- "stack_name": "HDP",
- "stack_version": "2.5"
- },
- "components": [
- {
- "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE",
- "StackServiceComponents": {
- "advertise_version": "true",
- "bulk_commands_display_name": "",
- "bulk_commands_master_component_name": "",
- "cardinality": "0-1",
- "component_category": "MASTER",
- "component_name": "HIVE_SERVER_INTERACTIVE",
- "custom_commands": ["RESTART_LLAP"],
- "decommission_allowed": "false",
- "display_name": "HiveServer2 Interactive",
- "has_bulk_commands_definition": "false",
- "is_client": "false",
- "is_master": "true",
- "reassign_allowed": "false",
- "recovery_enabled": "false",
- "service_name": "HIVE",
- "stack_name": "HDP",
- "stack_version": "2.5",
- "hostnames": ["c6401.ambari.apache.org"]
- },
- "dependencies": []
- },
- {
- "StackServiceComponents": {
- "advertise_version": "true",
- "cardinality": "1+",
- "component_category": "SLAVE",
- "component_name": "NODEMANAGER",
- "display_name": "NodeManager",
- "is_client": "false",
- "is_master": "false",
- "hostnames": [
- "c6403.ambari.apache.org"
- ]
- },
- "dependencies": []
- },
- ]
- }
+ "StackServices": {
+ "service_name": "TEZ"
+ }
+ },
+ {
+ "StackServices": {
+ "service_name": "SPARK"
+ }
+ },
+ {
+ "StackServices": {
+ "service_name": "YARN",
+ },
+ "Versions": {
+ "stack_version": "2.5"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "NODEMANAGER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ }
+ }
+ ]
+ }, {
+ "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE",
+ "StackServices": {
+ "service_name": "HIVE",
+ "service_version": "1.2.1.2.5",
+ "stack_name": "HDP",
+ "stack_version": "2.5"
+ },
+ "components": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE",
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "bulk_commands_display_name": "",
+ "bulk_commands_master_component_name": "",
+ "cardinality": "0-1",
+ "component_category": "MASTER",
+ "component_name": "HIVE_SERVER_INTERACTIVE",
+ "custom_commands": ["RESTART_LLAP"],
+ "decommission_allowed": "false",
+ "display_name": "HiveServer2 Interactive",
+ "has_bulk_commands_definition": "false",
+ "is_client": "false",
+ "is_master": "true",
+ "reassign_allowed": "false",
+ "recovery_enabled": "false",
+ "service_name": "HIVE",
+ "stack_name": "HDP",
+ "stack_version": "2.5",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "hostnames": [
+ "c6403.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ },
+ ]
+ }
],
"changed-configurations": [
{
@@ -898,6 +908,12 @@ class TestHDP25StackAdvisor(TestCase):
}
],
"configurations": {
+ "cluster-env": {
+ "properties": {
+ "stack_root": "{\"HDP\":\"/usr/hdp\"}",
+ "stack_name": "HDP"
+ },
+ },
"capacity-scheduler": {
"properties": {
"capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=60\n'
@@ -960,7 +976,8 @@ class TestHDP25StackAdvisor(TestCase):
"tez.am.resource.memory.mb": "341"
}
}
- }
+ },
+ "ambari-server-properties": {"ambari-server.user":"ambari_user"}
}
clusterData = {
@@ -990,6 +1007,9 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'default')
self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], 'default')
+ self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes'],
+ 'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl,org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin')
+ self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/${hdp.version}/spark/hdpLib/*')
self.assertTrue('hive-interactive-env' not in configurations)
self.assertTrue('property_attributes' not in configurations)