You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ec...@apache.org on 2017/02/24 09:18:51 UTC
ambari git commit: AMBARI-20126. Add support for Spark2 upgrade from
HDP-2.5 (dgrinenko via echekankiy)
Repository: ambari
Updated Branches:
refs/heads/trunk 043d6c06c -> 19da5823c
AMBARI-20126. Add support for Spark2 upgrade from HDP-2.5 (dgrinenko via echekankiy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/19da5823
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/19da5823
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/19da5823
Branch: refs/heads/trunk
Commit: 19da5823c8adc8cd18048040c93f43941c0e568f
Parents: 043d6c0
Author: Eugene Chekanskiy <ec...@hortonworks.com>
Authored: Fri Feb 24 11:18:33 2017 +0200
Committer: Eugene Chekanskiy <ec...@hortonworks.com>
Committed: Fri Feb 24 11:18:33 2017 +0200
----------------------------------------------------------------------
.../2.0.0/package/scripts/job_history_server.py | 2 +-
.../2.0.0/package/scripts/livy2_server.py | 2 +-
.../2.0.0/package/scripts/spark_client.py | 2 +-
.../package/scripts/spark_thrift_server.py | 9 ++--
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 11 +++++
.../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 50 +++++++++++++++++++
.../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 39 +++++++++++++++
.../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 51 ++++++++++++++++++++
.../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 41 ++++++++++++++++
9 files changed, 199 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py
index 154c83d..2631b49 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py
@@ -41,7 +41,7 @@ class JobHistoryServer(Script):
self.install_packages(env)
- def configure(self, env, upgrade_type=None):
+ def configure(self, env, upgrade_type=None, config_dir=None):
import params
env.set_params(params)
http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py
index 8c66998..cb4f5ee 100644
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py
@@ -44,7 +44,7 @@ class LivyServer(Script):
self.install_packages(env)
- def configure(self, env, upgrade_type=None):
+ def configure(self, env, upgrade_type=None, config_dir=None):
import params
env.set_params(params)
http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py
index 2c19b88..563b7e9 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py
@@ -34,7 +34,7 @@ class SparkClient(Script):
self.install_packages(env)
self.configure(env)
- def configure(self, env, upgrade_type=None):
+ def configure(self, env, upgrade_type=None, config_dir=None):
import params
env.set_params(params)
http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py
index 426c05c..72307cb 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py
@@ -40,7 +40,7 @@ class SparkThriftServer(Script):
self.install_packages(env)
- def configure(self, env ,upgrade_type=None):
+ def configure(self, env, upgrade_type=None, config_dir=None):
import params
env.set_params(params)
setup_spark(env, 'server', upgrade_type = upgrade_type, action = 'config')
@@ -69,10 +69,9 @@ class SparkThriftServer(Script):
import params
env.set_params(params)
- if params.version and check_stack_feature(StackFeature.SPARK2_THRIFTSERVER, params.version):
- Logger.info("Executing Spark2 Thrift Server Stack Upgrade pre-restart")
- conf_select.select(params.stack_name, "spark2", params.version)
- stack_select.select("spark2-thriftserver", params.version)
+ Logger.info("Executing Spark2 Thrift Server Stack Upgrade pre-restart")
+ conf_select.select(params.stack_name, "spark2", params.version)
+ stack_select.select("spark2-thriftserver", params.version)
def get_log_folder(self):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 3069b34..da334be 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -94,6 +94,17 @@
</component>
</service>
+ <service name="SPARK2">
+ <component name="SPARK2_CLIENT">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue">
+ <type>spark2-defaults</type>
+ <set key="spark.yarn.queue" value="default" if-type="spark-defaults" if-key="spark.yarn.queue" if-key-state="absent"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
<service name="TEZ">
<component name="TEZ_CLIENT">
<changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index 9516b6a..d396033 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -110,6 +110,11 @@
<component>LIVY_SERVER</component>
</service>
+ <service name="SPARK2">
+ <component>SPARK2_JOBHISTORYSERVER</component>
+ <component>SPARK2_THRIFTSERVER</component>
+ </service>
+
<service name="ZEPPELIN">
<component>ZEPPELIN_MASTER</component>
</service>
@@ -538,6 +543,12 @@
<summary>Add queue customization property</summary>
</task>
</execute-stage>
+
+ <execute-stage service="SPARK2" component="SPARK2_CLIENT" title="Apply config changes for Spark">
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue">
+ <summary>Add queue customization property</summary>
+ </task>
+ </execute-stage>
</group>
<!--
@@ -795,6 +806,17 @@
</service>
</group>
+ <group xsi:type="restart" name="SPARK2" title="Spark2">
+ <service-check>false</service-check>
+ <skippable>true</skippable>
+ <supports-auto-skip-failure>false</supports-auto-skip-failure>
+ <parallel-scheduler/>
+ <service name="SPARK2">
+ <component>SPARK2_JOBHISTORYSERVER</component>
+ <component>SPARK2_THRIFTSERVER</component>
+ </service>
+ </group>
+
<group xsi:type="restart" name="SPARK_CLIENTS" title="Spark Clients">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -804,6 +826,15 @@
</service>
</group>
+ <group xsi:type="restart" name="SPARK2_CLIENTS" title="Spark2 Clients">
+ <service-check>false</service-check>
+ <skippable>true</skippable>
+ <parallel-scheduler/>
+ <service name="SPARK2">
+ <component>SPARK2_CLIENT</component>
+ </service>
+ </group>
+
<group xsi:type="restart" name="ZEPPELIN" title="Zeppelin">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -860,6 +891,7 @@
<priority>
<service>HIVE</service>
<service>SPARK</service>
+ <service>SPARK2</service>
<service>SLIDER</service>
<service>ATLAS</service>
<service>ZEPPELIN</service>
@@ -1275,6 +1307,24 @@
</component>
</service>
+ <service name="SPARK2">
+ <component name="SPARK2_JOBHISTORYSERVER">
+ <upgrade>
+ <task xsi:type="restart-task"/>
+ </upgrade>
+ </component>
+ <component name="SPARK2_THRIFTSERVER">
+ <upgrade>
+ <task xsi:type="restart-task"/>
+ </upgrade>
+ </component>
+ <component name="SPARK2_CLIENT">
+ <upgrade>
+ <task xsi:type="restart-task"/>
+ </upgrade>
+ </component>
+ </service>
+
<service name="ZEPPELIN">
<component name="ZEPPELIN_MASTER">
<upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index 04d603b..c01d11e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -259,6 +259,16 @@
</service>
</group>
+ <group name="SPARK2" title="Spark2">
+ <skippable>true</skippable>
+ <supports-auto-skip-failure>false</supports-auto-skip-failure>
+ <service-check>false</service-check>
+ <service name="SPARK2">
+ <component>SPARK2_JOBHISTORYSERVER</component>
+ <component>SPARK2_THRIFTSERVER</component>
+ </service>
+ </group>
+
<group name="ZEPPELIN" title="Zeppelin">
<skippable>true</skippable>
<supports-auto-skip-failure>false</supports-auto-skip-failure>
@@ -275,6 +285,13 @@
</service>
</group>
+ <group name="SPARK2_CLIENTS" title="Spark2 Clients">
+ <skippable>true</skippable>
+ <service name="SPARK2">
+ <component>SPARK2_CLIENT</component>
+ </service>
+ </group>
+
<group name="ATLAS" title="Atlas">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -868,6 +885,28 @@
</component>
</service>
+ <service name="SPARK2">
+ <component name="SPARK2_JOBHISTORYSERVER">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ <component name="SPARK2_THRIFTSERVER">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ <component name="SPARK2_CLIENT">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue"/>
+ </pre-upgrade>
+ <pre-downgrade/>
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
+
<service name="ZEPPELIN">
<component name="ZEPPELIN_MASTER">
<upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
index 5347f01..921e479 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
@@ -109,6 +109,12 @@
<component>LIVY_SERVER</component>
</service>
+ <service name="SPARK2">
+ <component>SPARK2_JOBHISTORYSERVER</component>
+ <component>SPARK2_THRIFTSERVER</component>
+ <component>LIVY2_SERVER</component>
+ </service>
+
<service name="ZEPPELIN">
<component>ZEPPELIN_MASTER</component>
</service>
@@ -527,6 +533,18 @@
</service>
</group>
+ <group xsi:type="restart" name="SPARK2" title="Spark2">
+ <service-check>false</service-check>
+ <skippable>true</skippable>
+ <supports-auto-skip-failure>false</supports-auto-skip-failure>
+ <parallel-scheduler/>
+ <service name="SPARK2">
+ <component>SPARK2_JOBHISTORYSERVER</component>
+ <component>SPARK2_THRIFTSERVER</component>
+ <component>LIVY2_SERVER</component>
+ </service>
+ </group>
+
<group xsi:type="restart" name="SPARK_CLIENTS" title="Spark Clients">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -536,6 +554,15 @@
</service>
</group>
+ <group xsi:type="restart" name="SPARK2_CLIENTS" title="Spark2 Clients">
+ <service-check>false</service-check>
+ <skippable>true</skippable>
+ <parallel-scheduler/>
+ <service name="SPARK2">
+ <component>SPARK2_CLIENT</component>
+ </service>
+ </group>
+
<group xsi:type="restart" name="ZEPPELIN" title="Zeppelin">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -592,6 +619,7 @@
<priority>
<service>HIVE</service>
<service>SPARK</service>
+ <service>SPARK2</service>
<service>SLIDER</service>
<service>ATLAS</service>
<service>ZEPPELIN</service>
@@ -997,6 +1025,29 @@
</component>
</service>
+ <service name="SPARK2">
+ <component name="SPARK2_JOBHISTORYSERVER">
+ <upgrade>
+ <task xsi:type="restart-task"/>
+ </upgrade>
+ </component>
+ <component name="SPARK2_THRIFTSERVER">
+ <upgrade>
+ <task xsi:type="restart-task"/>
+ </upgrade>
+ </component>
+ <component name="LIVY2_SERVER">
+ <upgrade>
+ <task xsi:type="restart-task"/>
+ </upgrade>
+ </component>
+ <component name="SPARK2_CLIENT">
+ <upgrade>
+ <task xsi:type="restart-task"/>
+ </upgrade>
+ </component>
+ </service>
+
<service name="ZEPPELIN">
<component name="ZEPPELIN_MASTER">
<upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
index a4719c1..8669d2b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
@@ -258,6 +258,17 @@
</service>
</group>
+ <group name="SPARK2" title="Spark2">
+ <skippable>true</skippable>
+ <supports-auto-skip-failure>false</supports-auto-skip-failure>
+ <service-check>false</service-check>
+ <service name="SPARK2">
+ <component>SPARK2_JOBHISTORYSERVER</component>
+ <component>SPARK2_THRIFTSERVER</component>
+ <component>LIVY2_SERVER</component>
+ </service>
+ </group>
+
<group name="ZEPPELIN" title="Zeppelin">
<skippable>true</skippable>
<supports-auto-skip-failure>false</supports-auto-skip-failure>
@@ -274,6 +285,13 @@
</service>
</group>
+ <group name="SPARK2_CLIENTS" title="Spark2 Clients">
+ <skippable>true</skippable>
+ <service name="SPARK2">
+ <component>SPARK2_CLIENT</component>
+ </service>
+ </group>
+
<group name="ATLAS" title="Atlas">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -779,6 +797,29 @@
</component>
</service>
+ <service name="SPARK2">
+ <component name="SPARK2_JOBHISTORYSERVER">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ <component name="SPARK2_THRIFTSERVER">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ <component name="LIVY2_SERVER">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ <component name="SPARK2_CLIENT">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
+
<service name="ZEPPELIN">
<component name="ZEPPELIN_MASTER">
<upgrade>