You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2017/06/14 02:57:19 UTC
ambari git commit: AMBARI-21226. Add two new Spark thrift server
related configurations in Spark stack (Saisai Shao via smohanty)
Repository: ambari
Updated Branches:
refs/heads/trunk df685b92d -> 24e2cacb8
AMBARI-21226. Add two new Spark thrift server related configurations in Spark stack (Saisai Shao via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/24e2cacb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/24e2cacb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/24e2cacb
Branch: refs/heads/trunk
Commit: 24e2cacb8ede081824b9162ba124acdffa33e0ec
Parents: df685b9
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Jun 13 19:57:06 2017 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Jun 13 19:57:12 2017 -0700
----------------------------------------------------------------------
.../configuration/spark-thrift-sparkconf.xml | 40 ++++++++++++++++++++
.../configuration/spark2-thrift-sparkconf.xml | 40 ++++++++++++++++++++
.../stacks/HDP/2.6/upgrades/config-upgrade.xml | 24 ++++++++++++
.../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 26 +++++++++----
.../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 32 ++++++++++------
5 files changed, 142 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/24e2cacb/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml
new file mode 100644
index 0000000..001ef1e
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+ <property>
+ <name>spark.yarn.executor.failuresValidityInterval</name>
+ <value>2h</value>
+ <description>
+ Defines the validity interval for executor failure tracking.
+ Executor failures which are older than the validity interval will be ignored.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>spark.yarn.maxAppAttempts</name>
+ <value>1</value>
+ <description>
+ The maximum number of attempts that will be made to submit the application.
+ It should be no larger than the global number of max attempts in the YARN configuration.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/24e2cacb/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml
new file mode 100644
index 0000000..001ef1e
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+ <property>
+ <name>spark.yarn.executor.failuresValidityInterval</name>
+ <value>2h</value>
+ <description>
+ Defines the validity interval for executor failure tracking.
+ Executor failures which are older than the validity interval will be ignored.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>spark.yarn.maxAppAttempts</name>
+ <value>1</value>
+ <description>
+ The maximum number of attempts that will be made to submit the application.
+ It should be no larger than the global number of max attempts in the YARN configuration.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/24e2cacb/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index 1610bb5..564dead 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -168,5 +168,29 @@
</component>
</service>
+ <service name="SPARK">
+ <component name="SPARK_THRIFTSERVER">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_6_0_0_spark_thriftserver">
+ <type>spark-thrift-sparkconf</type>
+ <set key="spark.yarn.executor.failuresValidityInterval" value="2h"/>
+ <set key="spark.yarn.maxAppAttempts" value="1"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
+ <service name="SPARK2">
+ <component name="SPARK2_THRIFTSERVER">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_6_0_0_spark2_thriftserver">
+ <type>spark2-thrift-sparkconf</type>
+ <set key="spark.yarn.executor.failuresValidityInterval" value="2h"/>
+ <set key="spark.yarn.maxAppAttempts" value="1"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
</services>
</upgrade-config-changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/24e2cacb/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
index 1cdd184..6b827c9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
@@ -75,7 +75,7 @@
<service name="FLUME">
<component>FLUME_HANDLER</component>
</service>
-
+
<service name="ACCUMULO">
<component>ACCUMULO_TRACER</component>
<component>ACCUMULO_GC</component>
@@ -284,7 +284,7 @@
</execute-stage>
</group>
-
+
<group xsi:type="cluster" name="Upgrade service configs" title="Upgrade service configs">
<direction>UPGRADE</direction> <!-- prevent config changes on downgrade -->
<skippable>true</skippable> <!-- May fix configuration problems manually -->
@@ -322,7 +322,7 @@
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for Ranger Yarn plugin">
<task xsi:type="configure" id="hdp_2_6_maint_ranger_yarn_plugin_cluster_name"/>
</execute-stage>
-
+
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for YARN Preemption">
<task xsi:type="configure" id="hdp_2_6_yarn_preemption"/>
</execute-stage>
@@ -353,8 +353,18 @@
<task xsi:type="configure" id="hdp_2_6_maint_ranger_kms_plugin_cluster_name"/>
</execute-stage>
+ <!--SPARK-->
+ <execute-stage service="SPARK" component="SPARK_THRIFTSERVER" title="Apply config changes for Spark ThriftServer">
+ <task xsi:type="configure" id="hdp_2_6_0_0_spark_thriftserver"/>
+ </execute-stage>
+
+ <!--SPARK2-->
+ <execute-stage service="SPARK2" component="SPARK2_THRIFTSERVER" title="Apply config changes for Spark2 ThriftServer">
+ <task xsi:type="configure" id="hdp_2_6_0_0_spark2_thriftserver"/>
+ </execute-stage>
+
</group>
-
+
<!--
After processing this group, the user-specified Kerberos descriptor will be updated to work with
@@ -383,7 +393,7 @@
</task>
</execute-stage>
</group>
-
+
<!-- Now, restart all of the services. -->
<group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
<service-check>false</service-check>
@@ -747,7 +757,7 @@
<component>FLUME_HANDLER</component>
</service>
</group>
-
+
<group xsi:type="restart" name="ACCUMULO" title="Accumulo">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -790,7 +800,7 @@
<group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check">
<direction>UPGRADE</direction>
-
+
<execute-stage title="Check Component Versions">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" />
</execute-stage>
@@ -870,7 +880,7 @@
<function>setup_ranger_java_patches</function>
</task>
</pre-upgrade>
-
+
<pre-downgrade copy-upgrade="true" />
<upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/24e2cacb/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
index 3e7e3d7..9227669 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
@@ -168,7 +168,7 @@
<component>HBASE_MASTER</component>
</service>
</group>
-
+
<group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check">
<direction>UPGRADE</direction>
<skippable>true</skippable>
@@ -184,7 +184,7 @@
<service>LOGSEARCH</service>
</exclude>
</group>
-
+
<group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -192,7 +192,7 @@
<component>DATANODE</component>
<component>NFS_GATEWAY</component>
</service>
-
+
<service name="HBASE">
<component>HBASE_REGIONSERVER</component>
<component>PHOENIX_QUERY_SERVER</component>
@@ -208,7 +208,7 @@
<message>The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services.</message>
</batch>
</group>
-
+
<group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check">
<direction>UPGRADE</direction>
<skippable>true</skippable>
@@ -460,7 +460,7 @@
<group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check">
<direction>UPGRADE</direction>
-
+
<execute-stage title="Check Component Versions">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" />
</execute-stage>
@@ -469,7 +469,7 @@
<group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
<skippable>true</skippable>
<supports-auto-skip-failure>false</supports-auto-skip-failure>
-
+
<execute-stage title="Confirm Finalize">
<direction>UPGRADE</direction>
<task xsi:type="manual">
@@ -493,7 +493,7 @@
<function>finalize_rolling_upgrade</function>
</task>
</execute-stage>
-
+
<execute-stage title="Save Cluster State">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction">
</task>
@@ -544,7 +544,7 @@
<function>setup_ranger_java_patches</function>
</task>
</pre-upgrade>
-
+
<pre-downgrade copy-upgrade="true" />
<upgrade>
@@ -645,7 +645,7 @@
</upgrade>
</component>
</service>
-
+
<service name="SLIDER">
<component name="SLIDER">
<upgrade>
@@ -667,7 +667,7 @@
<summary>Verifying LZO codec path for mapreduce</summary>
</task>
</pre-upgrade>
-
+
<pre-downgrade copy-upgrade="true" />
<upgrade>
@@ -748,9 +748,9 @@
<summary>Verifying LZO codec path for Tez</summary>
</task>
</pre-upgrade>
-
+
<pre-downgrade copy-upgrade="true" />
-
+
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -830,6 +830,10 @@
</upgrade>
</component>
<component name="SPARK_THRIFTSERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_6_0_0_spark_thriftserver"/>
+ </pre-upgrade>
+ <pre-downgrade />
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -853,6 +857,10 @@
</upgrade>
</component>
<component name="SPARK2_THRIFTSERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_6_0_0_spark2_thriftserver"/>
+ </pre-upgrade>
+ <pre-downgrade />
<upgrade>
<task xsi:type="restart-task" />
</upgrade>