You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2017/02/24 00:46:36 UTC
ambari git commit: AMBARI-20109. HDP stack - enable shuffle
connection keep-alive. (Siddharth Seth via Swapan Shridhar).
Repository: ambari
Updated Branches:
refs/heads/trunk ef355ae0d -> cce9489c2
AMBARI-20109. HDP stack - enable shuffle connection keep-alive. (Siddharth Seth via Swapan Shridhar).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cce9489c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cce9489c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cce9489c
Branch: refs/heads/trunk
Commit: cce9489c2ed551940c70e9003fa421d4c9880e38
Parents: ef355ae
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Thu Feb 23 16:46:20 2017 -0800
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Thu Feb 23 16:46:20 2017 -0800
----------------------------------------------------------------------
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 24 +++++++++++++++++
.../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 20 +++++++++++++++
.../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 6 +++++
.../HIVE/configuration/tez-interactive-site.xml | 6 +++++
.../2.6/services/TEZ/configuration/tez-site.xml | 7 +++++
.../YARN/configuration-mapred/mapred-site.xml | 27 ++++++++++++++++++++
6 files changed, 90 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 3069b34..bca641c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -177,6 +177,14 @@
</definition>
</changes>
</component>
+ <component name="NODEMANAGER">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_6_0_0_enable_mapreduce_shuffle_connection_keepalive">
+ <type>mapred-site</type>
+ <set key="mapreduce.shuffle.connection-keep-alive.enable" value="true"/>
+ </definition>
+ </changes>
+ </component>
</service>
<service name="MAPREDUCE2">
@@ -354,6 +362,17 @@
</component>
</service>
+ <service name="TEZ">
+ <component name="TEZ_CLIENT">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_6_0_0_tez_client_enable_connection_keepalive">
+ <type>tez-site</type>
+ <set key="tez.runtime.shuffle.keep-alive.enabled" value="true"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
<service name="PIG">
<component name="PIG">
<changes>
@@ -433,6 +452,11 @@
<set key="llap_headroom_space" value="12288"/>
</definition>
+ <definition xsi:type="configure" id="hdp_2_6_0_0_tez_interactive_enable_connection_keepalive">
+ <type>tez-interactive-site</type>
+ <set key="tez.runtime.shuffle.keep-alive.enabled" value="true"/>
+ </definition>
+
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index 9516b6a..8641146 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -287,6 +287,13 @@
</task>
</execute-stage>
+ <!--YARN-->
+ <execute-stage service="YARN" component="NODEMANAGER" title="Enable ShuffleHandler keep-alive">
+ <task xsi:type="configure" id="hdp_2_6_0_0_enable_mapreduce_shuffle_connection_keepalive">
+ <summary>Enabling connection keep alive for the MapReduce Shuffle Handler</summary>
+ </task>
+ </execute-stage>
+
<!--Yarn-->
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Parameterizing Yarn Log4J Properties Resource Manager">
<task xsi:type="configure" id="yarn_log4j_parameterize">
@@ -334,6 +341,13 @@
</task>
</execute-stage>
+ <!--TEZ-->
+ <execute-stage service="TEZ" component="TEZ_CLIENT" title="Enable Tez Client Shuffle Connectio Keep-Alive">
+ <task xsi:type="configure" id="hdp_2_6_0_0_tez_client_enable_connection_keepalive">
+ <summary>Enabling connection keep alive for Tez shuffle client</summary>
+ </task>
+ </execute-stage>
+
<!--OOZIE-->
<execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation">
@@ -527,6 +541,12 @@
<summary>Updating the Hive Log4J2 properties to include parameterizations</summary>
</task>
</execute-stage>
+
+ <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Enable Tez-HiveInteractive Client Shuffle Connectio Keep-Alive">
+ <task xsi:type="configure" id="hdp_2_6_0_0_tez_interactive_enable_connection_keepalive">
+ <summary>Enabling connection keep alive for Tez-HiveInteractive shuffle client</summary>
+ </task>
+ </execute-stage>
<!-- SPARK -->
<execute-stage service="SPARK" component="LIVY_SERVER" title="Apply config changes for Livy Server">
http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index 04d603b..689c36b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -683,6 +683,10 @@
</component>
<component name="NODEMANAGER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_6_0_0_enable_mapreduce_shuffle_connection_keepalive"/>
+ </pre-upgrade>
+ <pre-downgrade />
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -802,6 +806,7 @@
<task xsi:type="configure" id="hive_log4j2_parameterize"/>
<task xsi:type="configure" id="llap_cli_log4j2_parameterize"/>
<task xsi:type="configure" id="llap_update_headroom"/>
+ <task xsi:type="configure" id="hdp_2_6_0_0_tez_interactive_enable_connection_keepalive"/>
</pre-upgrade>
<pre-downgrade />
@@ -898,6 +903,7 @@
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
+ <task xsi:type="configure" id="hdp_2_6_0_0_tez_client_enable_connection_keepalive"/>
</pre-upgrade>
<pre-downgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
index 3ce5fed..3bcb49a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
@@ -74,5 +74,11 @@
</description>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>tez.runtime.shuffle.keep-alive.enabled</name>
+ <value>true</value>
+ <description>Enables/Disables connection keep from the client side</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml
index 4bd1a19..463a2ed 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml
@@ -25,4 +25,11 @@
</description>
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>tez.runtime.shuffle.keep-alive.enabled</name>
+ <value>true</value>
+ <description>Enables/Disables connection keep from the client side</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cce9489c/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration-mapred/mapred-site.xml
new file mode 100644
index 0000000..1654c1c
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration-mapred/mapred-site.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Put site-specific property overrides in this file. -->
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude" supports_final="true">
+ <property>
+ <name>mapreduce.shuffle.connection-keep-alive.enable</name>
+ <value>true</value>
+ <description>Enable/Disable connection keep-alive for the MapReduce shuffle handler</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>