You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2016/05/31 01:05:13 UTC
ambari git commit: AMBARI-16857. Support yarn aux service classpath
isolation feature in spark (Bikas Saha via smohanty)
Repository: ambari
Updated Branches:
refs/heads/trunk 46b2fcdef -> f8e83d365
AMBARI-16857. Support yarn aux service classpath isolation feature in spark (Bikas Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f8e83d36
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f8e83d36
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f8e83d36
Branch: refs/heads/trunk
Commit: f8e83d365a87eb1651cfc7a5a1e48b7df6e5e6bf
Parents: 46b2fcd
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon May 30 17:45:02 2016 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon May 30 17:45:02 2016 -0700
----------------------------------------------------------------------
.../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 4 ++
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 3 ++
.../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 5 +++
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 3 ++
.../services/YARN/configuration/yarn-site.xml | 43 ++++++++++++++++++++
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 11 +++++
6 files changed, 69 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index 4362fe0..c7d8b30 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -291,6 +291,10 @@
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
</execute-stage>
+ <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
+ <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+ </execute-stage>
+
<execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
<task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
</execute-stage>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 0e0ea1d..e39f413 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -590,6 +590,9 @@
</component>
<component name="NODEMANAGER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+ </pre-upgrade>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
index 5ccb153..47e849c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
@@ -277,6 +277,11 @@
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
</execute-stage>
+ <!-- YARN -->
+ <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
+ <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+ </execute-stage>
+
<!--TEZ-->
<execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
<task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index bc45be6..644ca87 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -584,6 +584,9 @@
</component>
<component name="NODEMANAGER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+ </pre-upgrade>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
new file mode 100644
index 0000000..0f0e44f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration supports_final="true">
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle,spark_shuffle,spark2_shuffle</value>
+ <description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can
+ not start with numbers</description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.spark2_shuffle.class</name>
+ <value>org.apache.spark.network.yarn.YarnShuffleService</value>
+ <description>The auxiliary service class to use for Spark 2</description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
+ <value>/usr/hdp/${hdp.version}/spark/aux/*</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
+ <value>/usr/hdp/${hdp.version}/spark2/aux/*</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 5211276..aaa2d21 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -50,6 +50,17 @@
</changes>
</component>
</service>
+ <service name="YARN">
+ <component name="NODEMANAGER">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle">
+ <type>yarn-site</type>
+ <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle,spark2_shuffle"/>
+ <!-- Ideally we need to append spark2_shuffle to the existing value -->
+ </definition>
+ </changes>
+ </component>
+ </service>
</services>
</upgrade-config-changes>