You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/11/28 20:40:06 UTC
[05/12] ambari git commit: AMBARI-22525 - ATS start failed during
patch upgrade due to CNF SparkATSPlugin (jonathanhurley)
AMBARI-22525 - ATS start failed during patch upgrade due to CNF SparkATSPlugin (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2cf8cb85
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2cf8cb85
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2cf8cb85
Branch: refs/heads/branch-feature-AMBARI-22457
Commit: 2cf8cb85bffff266897e5941deaad2e92ae53447
Parents: 4fe795d
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Mon Nov 27 13:58:38 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Tue Nov 28 09:24:50 2017 -0500
----------------------------------------------------------------------
.../libraries/functions/component_version.py | 9 +++++----
.../YARN/2.1.0.2.0/package/scripts/params_linux.py | 6 ++++++
.../BigInsights/4.0/stack-advisor/stack_advisor_25.py | 2 +-
.../HDP/2.5/services/YARN/configuration/yarn-site.xml | 4 ++--
.../stacks/HDP/2.5/services/stack_advisor.py | 2 +-
.../stacks/HDP/2.6/upgrades/config-upgrade.xml | 14 ++++++++++++--
.../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 11 ++++++++---
.../resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 4 ++++
.../python/stacks/2.5/common/test_stack_advisor.py | 2 +-
9 files changed, 40 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
index 169b339..fd18232 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
@@ -20,7 +20,7 @@ limitations under the License.
from resource_management.libraries.script.script import Script
-def get_component_repository_version(service_name = None, component_name = None):
+def get_component_repository_version(service_name = None, component_name = None, default_value = None):
"""
Gets the version associated with the specified component from the structure in the command.
Every command should contain a mapping of service/component to the desired repository it's set
@@ -28,22 +28,23 @@ def get_component_repository_version(service_name = None, component_name = None)
:service_name: the name of the service
:component_name: the name of the component
+ :default_value: the value to return if either the service or the component are not found
"""
config = Script.get_config()
versions = _get_component_repositories(config)
if versions is None:
- return None
+ return default_value
if service_name is None:
service_name = config['serviceName'] if config is not None and 'serviceName' in config else None
if service_name is None or service_name not in versions:
- return None
+ return default_value
component_versions = versions[service_name]
if len(component_versions) == 0:
- return None
+ return default_value
if component_name is None:
component_name = config["role"] if config is not None and "role" in config else None
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 4079f18..2871dc0 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -84,6 +84,12 @@ stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_
# It cannot be used during the initial Cluser Install because the version is not yet known.
version = default("/commandParams/version", None)
+# these are used to render the classpath for picking up Spark classes
+# in the event that spark is not installed, then we must default to the vesrion of YARN installed
+# since it will still load classes from its own spark version
+spark_version = component_version.get_component_repository_version("SPARK", "SPARK_CLIENT", default_value = version)
+spark2_version = component_version.get_component_repository_version("SPARK2", "SPARK2_CLIENT", default_value = version)
+
stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py
index 7e77382..ead9e83 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py
@@ -745,7 +745,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if self.__isServiceDeployed(services, "SPARK"):
timeline_plugin_classes_values.append('org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin')
- timeline_plugin_classpath_values.append(stack_root + "/${hdp.version}/spark/hdpLib/*")
+ timeline_plugin_classpath_values.append(stack_root + "/{{spark_version}}/spark/hdpLib/*")
putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', ",".join(timeline_plugin_classes_values))
putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath', ":".join(timeline_plugin_classpath_values))
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
index b6fadcb..29833fb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
@@ -31,13 +31,13 @@
</property>
<property>
<name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
- <value>{{stack_root}}/${hdp.version}/spark/aux/*</value>
+ <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value>
<description>The auxiliary service classpath to use for Spark</description>
<on-ambari-upgrade add="false"/>
</property>
<property>
<name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
- <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value>
+ <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value>
<description>The auxiliary service classpath to use for Spark 2</description>
<on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index a3b8263..4187dc9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -824,7 +824,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if self.__isServiceDeployed(services, "SPARK"):
timeline_plugin_classes_values.append('org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin')
- timeline_plugin_classpath_values.append(stack_root + "/${hdp.version}/spark/hdpLib/*")
+ timeline_plugin_classpath_values.append(stack_root + "/{{spark_version}}/spark/hdpLib/*")
putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', ",".join(timeline_plugin_classes_values))
putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath', ":".join(timeline_plugin_classpath_values))
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index 4538072..42dc124 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -168,14 +168,13 @@
if-key-state="absent" />
</definition>
-
<definition xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" summary="Application Classpath Parameterization">
<type>yarn-site</type>
<replace key="yarn.application.classpath" find="/usr/hdp/current/hadoop-client" replace-with="{{hadoop_home}}"/>
</definition>
-
</changes>
</component>
+
<component name="NODEMANAGER">
<changes>
<definition xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem">
@@ -185,6 +184,17 @@
</definition>
</changes>
</component>
+
+ <component name="APP_TIMELINE_SERVER">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths">
+ <type>yarn-site</type>
+ <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/{{spark_version}}/spark/aux/*"/>
+ <set key="yarn.nodemanager.aux-services.spark2_shuffle.classpath" value="{{stack_root}}/{{spark2_version}}/spark2/aux/*"/>
+ <replace key="yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath" find="${hdp.version}/spark/" replace-with="{{spark_version}}/spark/"/>
+ </definition>
+ </changes>
+ </component>
</service>
<service name="KAFKA">
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
index f3cea48..2185049 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
@@ -146,7 +146,7 @@
<component>DRUID_OVERLORD</component>
<component>DRUID_SUPERSET</component>
</service>
-
+
<service name="RANGER">
<component>RANGER_USERSYNC</component>
<component>RANGER_ADMIN</component>
@@ -354,7 +354,12 @@
<task xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" supports-patch="true"/>
</execute-stage>
- <!-- YARN -->
+ <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Spark Versioned Classpaths">
+ <task xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths" supports-patch="true">
+ <summary>Updating the Spark classpath entries to use the specific version of Spark which is installed.</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="YARN" component="NODEMANAGER" title="Apply config changes for YARN NM">
<task xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem">
<summary>Updating YARN NodeManager config for LLAP</summary>
@@ -417,7 +422,7 @@
<execute-stage service="TEZ" component="TEZ_CLIENT" title="Updating LD Library Classpath for Tarball" >
<task xsi:type="configure" id="hdp_2_6_tez_tarball_ld_library" supports-patch="true"/>
</execute-stage>
-
+
<!-- MapR -->
<execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Updating LD Library Classpath for Tarball">
<task xsi:type="configure" id="hdp_2_6_mapreduce_tarball_ld_library" supports-patch="true"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
index 3bb6322..d51706a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
@@ -690,6 +690,10 @@
<service name="YARN">
<component name="APP_TIMELINE_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths" supports-patch="true"/>
+ </pre-upgrade>
+ <pre-downgrade/>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index bb11969..8f27d79 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -1206,7 +1206,7 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], 'default')
self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes'],
'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl,org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin')
- self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/${hdp.version}/spark/hdpLib/*')
+ self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/{{spark_version}}/spark/hdpLib/*')
self.assertTrue('hive-interactive-env' not in configurations)
self.assertTrue('property_attributes' not in configurations)