You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/11/29 16:58:24 UTC
[07/24] ambari git commit: AMBARI-22525 - ATS start failed during
patch upgrade due to CNF SparkATSPlugin (jonathanhurley)
AMBARI-22525 - ATS start failed during patch upgrade due to CNF SparkATSPlugin (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3fa1289e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3fa1289e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3fa1289e
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 3fa1289eaaf9c30985bcf5a50725080a9eb9b368
Parents: 393fdb8
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Mon Nov 27 13:58:38 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Tue Nov 28 10:24:31 2017 -0500
----------------------------------------------------------------------
.../libraries/functions/component_version.py | 9 +++++----
.../YARN/2.1.0.2.0/package/scripts/params_linux.py | 6 ++++++
.../YARN/3.0.0.3.0/package/scripts/params_linux.py | 6 ++++++
.../HDP/2.5/services/YARN/configuration/yarn-site.xml | 4 ++--
.../stacks/HDP/2.5/services/stack_advisor.py | 2 +-
.../stacks/HDP/2.6/upgrades/config-upgrade.xml | 14 ++++++++++++--
.../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 10 ++++++++--
.../resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 4 ++++
.../HDP/3.0/services/YARN/configuration/yarn-site.xml | 4 ++--
.../1.0/services/FAKEYARN/configuration/yarn-site.xml | 4 ++--
.../python/stacks/2.5/common/test_stack_advisor.py | 2 +-
11 files changed, 49 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
index 169b339..fd18232 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
@@ -20,7 +20,7 @@ limitations under the License.
from resource_management.libraries.script.script import Script
-def get_component_repository_version(service_name = None, component_name = None):
+def get_component_repository_version(service_name = None, component_name = None, default_value = None):
"""
Gets the version associated with the specified component from the structure in the command.
Every command should contain a mapping of service/component to the desired repository it's set
@@ -28,22 +28,23 @@ def get_component_repository_version(service_name = None, component_name = None)
:service_name: the name of the service
:component_name: the name of the component
+ :default_value: the value to return if either the service or the component are not found
"""
config = Script.get_config()
versions = _get_component_repositories(config)
if versions is None:
- return None
+ return default_value
if service_name is None:
service_name = config['serviceName'] if config is not None and 'serviceName' in config else None
if service_name is None or service_name not in versions:
- return None
+ return default_value
component_versions = versions[service_name]
if len(component_versions) == 0:
- return None
+ return default_value
if component_name is None:
component_name = config["role"] if config is not None and "role" in config else None
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 3e4504d..b244af0 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -84,6 +84,12 @@ stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_
# It cannot be used during the initial Cluser Install because the version is not yet known.
version = default("/commandParams/version", None)
+# these are used to render the classpath for picking up Spark classes
+# in the event that spark is not installed, then we must default to the vesrion of YARN installed
+# since it will still load classes from its own spark version
+spark_version = component_version.get_component_repository_version("SPARK", "SPARK_CLIENT", default_value = version)
+spark2_version = component_version.get_component_repository_version("SPARK2", "SPARK2_CLIENT", default_value = version)
+
stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
index 617dc3b..372ae85 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
@@ -84,6 +84,12 @@ stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_
# It cannot be used during the initial Cluser Install because the version is not yet known.
version = default("/commandParams/version", None)
+# these are used to render the classpath for picking up Spark classes
+# in the event that spark is not installed, then we must default to the vesrion of YARN installed
+# since it will still load classes from its own spark version
+spark_version = component_version.get_component_repository_version("SPARK", "SPARK_CLIENT", default_value = version)
+spark2_version = component_version.get_component_repository_version("SPARK2", "SPARK2_CLIENT", default_value = version)
+
stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
index b6fadcb..29833fb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
@@ -31,13 +31,13 @@
</property>
<property>
<name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
- <value>{{stack_root}}/${hdp.version}/spark/aux/*</value>
+ <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value>
<description>The auxiliary service classpath to use for Spark</description>
<on-ambari-upgrade add="false"/>
</property>
<property>
<name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
- <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value>
+ <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value>
<description>The auxiliary service classpath to use for Spark 2</description>
<on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index b6f2478..6f409cd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -823,7 +823,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if self.isServiceDeployed(services, "SPARK"):
timeline_plugin_classes_values.append('org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin')
- timeline_plugin_classpath_values.append(stack_root + "/${hdp.version}/spark/hdpLib/*")
+ timeline_plugin_classpath_values.append(stack_root + "/{{spark_version}}/spark/hdpLib/*")
putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', ",".join(timeline_plugin_classes_values))
putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath', ":".join(timeline_plugin_classpath_values))
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index bcb13bc..9311877 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -162,14 +162,13 @@
if-key-state="absent" />
</definition>
-
<definition xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" summary="Application Classpath Parameterization">
<type>yarn-site</type>
<replace key="yarn.application.classpath" find="/usr/hdp/current/hadoop-client" replace-with="{{hadoop_home}}"/>
</definition>
-
</changes>
</component>
+
<component name="NODEMANAGER">
<changes>
<definition xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem">
@@ -179,6 +178,17 @@
</definition>
</changes>
</component>
+
+ <component name="APP_TIMELINE_SERVER">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths">
+ <type>yarn-site</type>
+ <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/{{spark_version}}/spark/aux/*"/>
+ <set key="yarn.nodemanager.aux-services.spark2_shuffle.classpath" value="{{stack_root}}/{{spark2_version}}/spark2/aux/*"/>
+ <replace key="yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath" find="${hdp.version}/spark/" replace-with="{{spark_version}}/spark/"/>
+ </definition>
+ </changes>
+ </component>
</service>
<service name="KAFKA">
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
index 0355362..2184060 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
@@ -147,6 +147,7 @@
<component>DRUID_SUPERSET</component>
</service>
+
</group>
<group xsi:type="cluster" name="Backups" title="Perform Backups">
@@ -351,7 +352,12 @@
<task xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" supports-patch="true"/>
</execute-stage>
- <!-- YARN -->
+ <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Spark Versioned Classpaths">
+ <task xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths" supports-patch="true">
+ <summary>Updating the Spark classpath entries to use the specific version of Spark which is installed.</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="YARN" component="NODEMANAGER" title="Apply config changes for YARN NM">
<task xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem">
<summary>Updating YARN NodeManager config for LLAP</summary>
@@ -414,7 +420,7 @@
<execute-stage service="TEZ" component="TEZ_CLIENT" title="Updating LD Library Classpath for Tarball" >
<task xsi:type="configure" id="hdp_2_6_tez_tarball_ld_library" supports-patch="true"/>
</execute-stage>
-
+
<!-- MapR -->
<execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Updating LD Library Classpath for Tarball">
<task xsi:type="configure" id="hdp_2_6_mapreduce_tarball_ld_library" supports-patch="true"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
index 5aa2d20..70f84ab 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
@@ -690,6 +690,10 @@
<service name="YARN">
<component name="APP_TIMELINE_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths" supports-patch="true"/>
+ </pre-upgrade>
+ <pre-downgrade/>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
index 01551c4..d32e2de 100644
--- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
@@ -20,7 +20,7 @@
<property>
<name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
- <value>{{stack_root}}/${hdp.version}/spark/aux/*</value>
+ <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value>
<description>The auxiliary service classpath to use for Spark</description>
<on-ambari-upgrade add="false"/>
</property>
@@ -41,7 +41,7 @@
<!-- These configs were inherited from HDP 2.5 -->
<property>
<name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
- <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value>
+ <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value>
<description>The auxiliary service classpath to use for Spark 2</description>
<on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml
index 57d03a8..392d0de 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml
@@ -747,13 +747,13 @@
</property>
<property>
<name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
- <value>{{stack_root}}/${hdp.version}/spark/aux/*</value>
+ <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value>
<description>The auxiliary service classpath to use for Spark</description>
<on-ambari-upgrade add="false"/>
</property>
<property>
<name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
- <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value>
+ <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value>
<description>The auxiliary service classpath to use for Spark 2</description>
<on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 407e78d..cc6bd9d 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -1206,7 +1206,7 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], 'default')
self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes'],
'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl,org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin')
- self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/${hdp.version}/spark/hdpLib/*')
+ self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/{{spark_version}}/spark/hdpLib/*')
self.assertTrue('hive-interactive-env' not in configurations)
self.assertTrue('property_attributes' not in configurations)