You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2016/10/20 04:54:09 UTC
ambari git commit: AMBARI-17489. Remove
spark.yarn.max.executor.failures configuration in Spark Ambari definition
(Saisai Shao via smohanty)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 47affcdba -> 45ccfaf01
AMBARI-17489. Remove spark.yarn.max.executor.failures configuration in Spark Ambari definition (Saisai Shao via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/45ccfaf0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/45ccfaf0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/45ccfaf0
Branch: refs/heads/branch-2.4
Commit: 45ccfaf01a1607a9813b79d39fb7c02a882ab6f3
Parents: 47affcd
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Oct 19 21:54:05 2016 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Oct 19 21:54:05 2016 -0700
----------------------------------------------------------------------
.../1.6.0/configuration/spark-defaults.xml | 5 +++++
.../stacks/HDP/2.3/upgrades/config-upgrade.xml | 2 ++
.../stacks/HDP/2.4/upgrades/config-upgrade.xml | 22 +++++++++++++++++++-
.../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 20 +++++++++++++-----
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 10 +++++++--
5 files changed, 51 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/45ccfaf0/ambari-server/src/main/resources/common-services/SPARK/1.6.0/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.6.0/configuration/spark-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK/1.6.0/configuration/spark-defaults.xml
index 6465b72..0bcf10f 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.6.0/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.6.0/configuration/spark-defaults.xml
@@ -25,6 +25,11 @@
<on-ambari-upgrade add="false"/>
</property>
<property>
+ <name>spark.yarn.max.executor.failures</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>spark.history.provider</name>
<value>org.apache.spark.deploy.history.FsHistoryProvider</value>
<description>Name of history provider class</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/45ccfaf0/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index cdbe319..b63115b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -370,6 +370,7 @@
<transfer operation="delete" delete-key="spark.yarn.services" />
<transfer operation="delete" delete-key="spark.driver.extraJavaOptions" />
<transfer operation="delete" delete-key="spark.yarn.am.extraJavaOptions" />
+ <transfer operation="delete" delete-key="spark.yarn.max.executor.failures" />
<set key="spark.history.provider" value="org.apache.spark.deploy.history.FsHistoryProvider"/>
</definition>
<definition xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts">
@@ -384,6 +385,7 @@
<type>spark-defaults</type>
<transfer operation="delete" delete-key="spark.driver.extraJavaOptions" />
<transfer operation="delete" delete-key="spark.yarn.am.extraJavaOptions" />
+ <transfer operation="delete" delete-key="spark.yarn.max.executor.failures" />
</definition>
<definition xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts">
<type>spark-javaopts-properties</type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/45ccfaf0/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index 9045443..8a9996c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -282,7 +282,7 @@
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
-
+
<definition xsi:type="configure" id="hdp_2_5_0_0_upgrade_storm_1.0">
<type>storm-site</type>
<replace key="_storm.thrift.nonsecure.transport" find="backtype.storm.security.auth.SimpleTransportPlugin"
@@ -381,6 +381,26 @@
</changes>
</component>
</service>
+
+ <service name="SPARK">
+ <component name="SPARK_JOBHISTORYSERVER">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_5_0_0_spark_jobhistoryserver">
+ <type>spark-defaults</type>
+ <transfer operation="delete" delete-key="spark.yarn.max.executor.failures" />
+ </definition>
+ </changes>
+ </component>
+ <component name="SPARK_CLIENT">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_5_0_0_spark_client">
+ <type>spark-defaults</type>
+ <transfer operation="delete" delete-key="spark.yarn.max.executor.failures" />
+ </definition>
+ </changes>
+ </component>
+ </service>
+
</services>
</upgrade-config-changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/45ccfaf0/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
index 3478603..2924455 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
@@ -32,6 +32,7 @@
<check>org.apache.ambari.server.checks.AtlasPresenceCheck</check>
</prerequisite-checks>
+
<order>
<group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Upgrade">
<direction>UPGRADE</direction>
@@ -86,7 +87,7 @@
<component>ACCUMULO_MONITOR</component>
<component>ACCUMULO_MASTER</component>
</service>
-
+
<service name="STORM">
<component>DRPC_SERVER</component>
<component>STORM_UI_SERVER</component>
@@ -402,6 +403,15 @@
<execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger KMS Server">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db"/>
</execute-stage>
+
+ <!-- SPARK -->
+ <execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config changes for Spark JobHistoryServer">
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark_jobhistoryserver"/>
+ </execute-stage>
+ <execute-stage service="SPARK" component="SPARK_CLIENT" title="Apply config changes for Spark">
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark_client"/>
+ </execute-stage>
+
</group>
<!--
@@ -588,7 +598,7 @@
<service>HBASE</service>
</priority>
</group>
-
+
<group xsi:type="restart" name="HIVE_MASTERS" title="Hive Masters">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -663,7 +673,7 @@
<service>SPARK</service>
</priority>
</group>
-
+
<group xsi:type="restart" name="FALCON" title="Falcon">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -727,7 +737,7 @@
<component>FLUME_HANDLER</component>
</service>
</group>
-
+
<group xsi:type="restart" name="ACCUMULO" title="Accumulo">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -757,7 +767,7 @@
<group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check">
<direction>UPGRADE</direction>
-
+
<execute-stage title="Check Component Versions">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" />
</execute-stage>
http://git-wip-us.apache.org/repos/asf/ambari/blob/45ccfaf0/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index 826cf16..e4f56e9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -503,7 +503,7 @@
<message>-- If keytabs are managed manually, create the required ranger keytabs in the locations specified in ranger config and restart Ranger components.</message>
</task>
</execute-stage>
-
+
<execute-stage service="KAFKA" component="KAFKA_BROKER" title="Kafka Property Removal">
<task xsi:type="manual">
<message>Kafka Brokers have been upgraded with version compatability.</message>
@@ -653,7 +653,7 @@
<task xsi:type="restart-task" />
</upgrade>
</component>
-
+
<component name="NFS_GATEWAY">
<upgrade>
<task xsi:type="restart-task" />
@@ -876,6 +876,9 @@
<service name="SPARK">
<component name="SPARK_JOBHISTORYSERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark_jobhistoryserver"/>
+ </pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
<upgrade>
<task xsi:type="restart-task" />
@@ -888,6 +891,9 @@
</upgrade>
</component>
<component name="SPARK_CLIENT">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark_client"/>
+ </pre-upgrade>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>