You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/10/30 19:37:14 UTC
ambari git commit: AMBARI-13644. Stop-and-Start Upgrade: Handle Knox
in all upgrade paths and downgrades (alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk 467f846e5 -> 4ed0a03a4
AMBARI-13644. Stop-and-Start Upgrade: Handle Knox in all upgrade paths and downgrades (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4ed0a03a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4ed0a03a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4ed0a03a
Branch: refs/heads/trunk
Commit: 4ed0a03a4d0e1acb6648c4c8c26dc9a632e8c20c
Parents: 467f846
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Thu Oct 29 19:01:07 2015 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Fri Oct 30 11:36:09 2015 -0700
----------------------------------------------------------------------
.../HDFS/2.1.0.2.0/package/scripts/snamenode.py | 14 ++-
.../0.5.0.2.2/package/scripts/knox_gateway.py | 13 ++-
.../package/scripts/setup_ranger_knox.py | 4 +-
.../KNOX/0.5.0.2.2/package/scripts/upgrade.py | 2 +
.../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 70 ++++++------
.../HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml | 29 +++--
.../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml | 113 +++++++++----------
.../HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml | 29 +++--
.../stacks/2.1/FALCON/test_falcon_client.py | 4 +-
.../stacks/2.1/FALCON/test_falcon_server.py | 4 +-
.../python/stacks/2.2/KNOX/test_knox_gateway.py | 16 +--
.../stacks/2.3/MAHOUT/test_mahout_client.py | 4 +-
12 files changed, 156 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py
index f67b1cb..1748955 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py
@@ -18,14 +18,20 @@ limitations under the License.
"""
from resource_management import *
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import hdp_select
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
from resource_management.libraries.functions.security_commons import build_expectations, \
cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
FILE_TYPE_XML
+
from hdfs_snamenode import snamenode
from hdfs import hdfs
from ambari_commons.os_family_impl import OsFamilyImpl
from ambari_commons import OSConst
+from resource_management.core.logger import Logger
+
class SNameNode(Script):
def install(self, env):
import params
@@ -61,7 +67,13 @@ class SNameNodeDefault(SNameNode):
return {"HDP": "hadoop-hdfs-secondarynamenode"}
def pre_upgrade_restart(self, env, upgrade_type=None):
- pass
+ Logger.info("Executing Stack Upgrade pre-restart")
+ import params
+ env.set_params(params)
+
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ conf_select.select(params.stack_name, "hadoop", params.version)
+ hdp_select.select("hadoop-hdfs-secondarynamenode", params.version)
def security_status(self, env):
import status_params
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
index 7323a73..8d6d7d8 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
@@ -48,6 +48,9 @@ from knox_ldap import ldap
from setup_ranger_knox import setup_ranger_knox
class KnoxGateway(Script):
+ def get_stack_to_component(self):
+ return {"HDP": "knox-server"}
+
def install(self, env):
import params
env.set_params(params)
@@ -104,10 +107,8 @@ class KnoxGatewayWindows(KnoxGateway):
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
class KnoxGatewayDefault(KnoxGateway):
- def get_stack_to_component(self):
- return {"HDP": "knox-server"}
- def pre_rolling_restart(self, env):
+ def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
@@ -140,13 +141,13 @@ class KnoxGatewayDefault(KnoxGateway):
action = "delete",
)
- def start(self, env, rolling_restart=False):
+ def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env)
daemon_cmd = format('{knox_bin} start')
no_op_test = format('ls {knox_pid_file} >/dev/null 2>&1 && ps -p `cat {knox_pid_file}` >/dev/null 2>&1')
- setup_ranger_knox(rolling_upgrade=rolling_restart)
+ setup_ranger_knox(upgrade_type=upgrade_type)
# Used to setup symlink, needed to update the knox managed symlink, in case of custom locations
if os.path.islink(params.knox_managed_pid_symlink):
Link(params.knox_managed_pid_symlink,
@@ -159,7 +160,7 @@ class KnoxGatewayDefault(KnoxGateway):
not_if=no_op_test
)
- def stop(self, env, rolling_restart=False):
+ def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
daemon_cmd = format('{knox_bin} stop')
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
index 1efe9e0..8ea1427 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py
@@ -19,7 +19,7 @@ limitations under the License.
"""
from resource_management.core.logger import Logger
-def setup_ranger_knox(rolling_upgrade = False):
+def setup_ranger_knox(upgrade_type=None):
import params
if params.has_ranger_admin:
@@ -30,7 +30,7 @@ def setup_ranger_knox(rolling_upgrade = False):
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
hdp_version = None
- if rolling_upgrade:
+ if upgrade_type is not None:
hdp_version = params.version
if params.retryAble:
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
index a9bb9e9..55c5060 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
@@ -42,6 +42,8 @@ def backup_data():
Logger.info('Backing up Knox data directory before upgrade...')
directoryMappings = _get_directory_mappings_during_upgrade()
+ Logger.info("Directory mappings to backup: {0}".format(str(directoryMappings)))
+
absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)
if not os.path.isdir(absolute_backup_dir):
os.makedirs(absolute_backup_dir)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
index 8675f69..11b7be2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
@@ -197,46 +197,46 @@
<!--Changes for stack 2.2-->
<!--YARN-->
- <execute-stage service="MAPREDUCE2" component="HISTORYSERVER">
+ <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for HistoryServer">
<task xsi:type="configure" id="hdp_2_2_0_0_historyserver_classpath"/>
</execute-stage>
<!--Changes for stack 2.3-->
<!--HDFS-->
- <execute-stage service="HDFS" component="NAMENODE">
+ <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
<task xsi:type="configure" id="hdp_2_3_0_0_modify_hadoop_env"/>
</execute-stage>
<!--YARN-->
- <execute-stage service="MAPREDUCE2" component="HISTORYSERVER">
+ <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for HistoryServer">
<task xsi:type="configure" id="hdp_2_3_0_0_mapreduce2_adjust_history_server"/>
</execute-stage>
- <execute-stage service="YARN" component="APP_TIMELINE_SERVER">
+ <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Apply config changes for AppTimelineServer">
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_ats_enable_recovery"/>
</execute-stage>
- <execute-stage service="YARN" component="RESOURCEMANAGER">
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_disable_node_labels"/>
</execute-stage>
- <execute-stage service="YARN" component="RESOURCEMANAGER">
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_clear_default_node_label_expression"/>
</execute-stage>
- <execute-stage service="YARN" component="RESOURCEMANAGER">
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_def_capacity"/>
</execute-stage>
- <execute-stage service="YARN" component="RESOURCEMANAGER">
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
</execute-stage>
<!--HBASE-->
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_set_global_memstore_size"/>
</execute-stage>
@@ -244,27 +244,27 @@
<task xsi:type="server_action" summary="Calculating HBase Properties" class="org.apache.ambari.server.serveraction.upgrades.HBaseConfigCalculation" />
</execute-stage>
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_authorization_coprocessors"/>
</execute-stage>
<!--TEZ-->
- <execute-stage service="TEZ" component="TEZ_CLIENT">
+ <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
<task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties"/>
</execute-stage>
<!--HIVE-->
- <execute-stage service="HIVE" component="HIVE_SERVER">
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager"/>
</execute-stage>
- <execute-stage service="HIVE" component="HIVE_SERVER">
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/>
</execute-stage>
- <execute-stage service="HIVE" component="WEBHCAT_SERVER">
+ <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for WebHCat Server">
<task xsi:type="configure" id="hdp_2_3_0_0_webhcat_server_update_env"/>
</execute-stage>
@@ -274,51 +274,35 @@
<!--OOZIE-->
- <execute-stage service="OOZIE" component="OOZIE_SERVER">
+ <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
<task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
</execute-stage>
<!--STORM-->
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_nimbus_convert_nimbus_host_to_seeds"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_update_storm_env"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_add_storm_cluster_logs_content"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_add_storm_worker_logs_content"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_nimbus_update_env_vars"/>
</execute-stage>
</group>
-
- <!-- Invoke hdp-select set all after all components have been stopped by
- before starting new ones. This step must be done here in order to
- ensure that components which rely on other components (like ZKFC on hadoop)
- will start on the correct versions -->
- <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
- <skippable>true</skippable>
- <execute-stage title="Update stack to {{version}}">
- <task xsi:type="execute">
- <script>scripts/ru_set_all.py</script>
- <function>actionexecute</function>
- </task>
- </execute-stage>
- </group>
-
<!-- Now, restart all of the services. -->
-
<group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -463,6 +447,20 @@
</service>
</group>
+ <!--
+ Invoke "hdp-select set all" to change any components we may have missed
+ that are installed on the hosts but not known by Ambari.
+ -->
+ <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+ <skippable>true</skippable>
+ <execute-stage title="Update stack to {{version}}">
+ <task xsi:type="execute">
+ <script>scripts/ru_set_all.py</script>
+ <function>actionexecute</function>
+ </task>
+ </execute-stage>
+ </group>
+
<group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
<skippable>true</skippable>
<execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts">
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
index 7778f8e..f23e136 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
@@ -232,22 +232,7 @@
</execute-stage>
</group>
- <!-- Invoke hdp-select set all after all components have been stopped by
- before starting new ones. This step must be done here in order to
- ensure that components which rely on other components (like ZKFC on hadoop)
- will start on the correct versions -->
- <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
- <skippable>true</skippable>
- <execute-stage title="Update stack to {{version}}">
- <task xsi:type="execute">
- <script>scripts/ru_set_all.py</script>
- <function>actionexecute</function> <!-- TODO, parallelize -->
- </task>
- </execute-stage>
- </group>
-
<!-- Now, restart all of the services. -->
-
<group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -460,6 +445,20 @@
</service>
</group>
+ <!--
+ Invoke "hdp-select set all" to change any components we may have missed
+ that are installed on the hosts but not known by Ambari.
+ -->
+ <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+ <skippable>true</skippable>
+ <execute-stage title="Update stack to {{version}}">
+ <task xsi:type="execute">
+ <script>scripts/ru_set_all.py</script>
+ <function>actionexecute</function>
+ </task>
+ </execute-stage>
+ </group>
+
<group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
<skippable>true</skippable>
<execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts">
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
index cb5e32e..ff995ab 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
@@ -296,66 +296,66 @@
</execute-stage>
<!--HDFS-->
- <execute-stage service="HDFS" component="NAMENODE">
+ <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
<task xsi:type="configure" id="hdp_2_3_0_0_modify_hadoop_env"/>
</execute-stage>
- <execute-stage service="HDFS" component="NAMENODE">
+ <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
<task xsi:type="configure" id="hdp_2_3_0_0_hdfs_adjust_ranger_plugin"/>
</execute-stage>
- <execute-stage service="HDFS" component="NAMENODE">
+ <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
<task xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_policy"/>
</execute-stage>
- <execute-stage service="HDFS" component="NAMENODE">
+ <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
<task xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_audit"/>
</execute-stage>
- <execute-stage service="HDFS" component="NAMENODE">
+ <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
<task xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_security"/>
</execute-stage>
- <execute-stage service="HDFS" component="NAMENODE">
+ <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
<task xsi:type="configure" id="hdp_2_3_0_0_hdfs_ranger_hdfs_delete_old_properties"/>
</execute-stage>
<!--YARN-->
- <execute-stage service="MAPREDUCE2" component="HISTORYSERVER">
+ <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for HistoryServer">
<task xsi:type="configure" id="hdp_2_3_0_0_mapreduce2_adjust_history_server"/>
</execute-stage>
- <execute-stage service="YARN" component="APP_TIMELINE_SERVER">
+ <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Apply config changes for AppTimelineServer">
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_ats_enable_recovery"/>
</execute-stage>
- <execute-stage service="YARN" component="RESOURCEMANAGER">
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_disable_node_labels"/>
</execute-stage>
- <execute-stage service="YARN" component="RESOURCEMANAGER">
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_clear_default_node_label_expression"/>
</execute-stage>
- <execute-stage service="YARN" component="RESOURCEMANAGER">
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_def_capacity"/>
</execute-stage>
- <execute-stage service="YARN" component="RESOURCEMANAGER">
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for ResourceManager">
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
</execute-stage>
<!--HBASE-->
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_scheduler_factory"/>
</execute-stage>
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_rpc_controller_factory"/>
</execute-stage>
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_set_global_memstore_size"/>
</execute-stage>
@@ -363,141 +363,126 @@
<task xsi:type="server_action" summary="Calculating HBase Properties" class="org.apache.ambari.server.serveraction.upgrades.HBaseConfigCalculation" />
</execute-stage>
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_indexed_wal_edit_codec"/>
</execute-stage>
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_authorization_coprocessors"/>
</execute-stage>
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_transition_ranger_hbase_policy"/>
</execute-stage>
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_transition_ranger_hbase_audit"/>
</execute-stage>
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_copy_ranger_policies"/>
</execute-stage>
- <execute-stage service="HBASE" component="HBASE_MASTER">
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_delete_old_ranger_properties"/>
</execute-stage>
<!--TEZ-->
- <execute-stage service="TEZ" component="TEZ_CLIENT">
+ <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
<task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties"/>
</execute-stage>
<!--HIVE-->
- <execute-stage service="HIVE" component="HIVE_SERVER">
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager"/>
</execute-stage>
- <execute-stage service="HIVE" component="HIVE_SERVER">
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/>
</execute-stage>
- <execute-stage service="HIVE" component="HIVE_SERVER">
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_ranger_policy"/>
</execute-stage>
- <execute-stage service="HIVE" component="HIVE_SERVER">
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_ranger_security"/>
</execute-stage>
- <execute-stage service="HIVE" component="HIVE_SERVER">
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_ranger_audit"/>
</execute-stage>
- <execute-stage service="HIVE" component="HIVE_SERVER">
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_3_0_0_hive_server_remove_deprecated_ranger_properties"/>
</execute-stage>
- <execute-stage service="HIVE" component="WEBHCAT_SERVER">
+ <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for WebHCat Server">
<task xsi:type="configure" id="hdp_2_3_0_0_webhcat_server_update_env"/>
</execute-stage>
- <execute-stage service="HIVE" component="WEBHCAT_SERVER">
+ <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for WebHCat Server">
<task xsi:type="configure" id="hdp_2_3_0_0_webhcat_server_update_configuration_paths"/>
</execute-stage>
<!--OOZIE-->
- <execute-stage service="OOZIE" component="OOZIE_SERVER">
+ <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
<task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
</execute-stage>
<!--KNOX-->
- <execute-stage service="KNOX" component="KNOX_GATEWAY">
+ <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway">
<task xsi:type="configure" id="hdp_2_3_0_0_knox_configure_ranger_policy"/>
</execute-stage>
- <execute-stage service="KNOX" component="KNOX_GATEWAY">
+ <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway">
<task xsi:type="configure" id="hdp_2_3_0_0_knox_configure_ranger_knox_audit"/>
</execute-stage>
- <execute-stage service="KNOX" component="KNOX_GATEWAY">
+ <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway">
<task xsi:type="configure" id="hdp_2_3_0_0_knox_remove_deprecated_ranger_properties"/>
</execute-stage>
<!--STORM-->
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_nimbus_convert_nimbus_host_to_seeds"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_update_storm_env"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_add_storm_cluster_logs_content"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_add_storm_worker_logs_content"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_nimbus_update_env_vars"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_nimbus_configure_ranger_policy"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_nimbus_configure_ranger_audit"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS">
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_3_0_0_nimbus_remove_deprecated_ranger_properties"/>
</execute-stage>
</group>
- <!-- Invoke hdp-select set all after all components have been stopped but
- before starting new ones. This step must be done here in order to
- ensure that components which rely on other components (like ZKFC on hadoop)
- will start on the correct versions -->
- <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
- <skippable>true</skippable>
- <execute-stage title="Update stack to {{version}}">
- <task xsi:type="execute">
- <script>scripts/ru_set_all.py</script>
- <function>actionexecute</function>
- </task>
- </execute-stage>
- </group>
-
<!-- Now, restart all of the services. -->
-
<group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -722,6 +707,20 @@
</service>
</group>
+ <!--
+ Invoke "hdp-select set all" to change any components we may have missed
+ that are installed on the hosts but not known by Ambari.
+ -->
+ <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+ <skippable>true</skippable>
+ <execute-stage title="Update stack to {{version}}">
+ <task xsi:type="execute">
+ <script>scripts/ru_set_all.py</script>
+ <function>actionexecute</function>
+ </task>
+ </execute-stage>
+ </group>
+
<group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
<skippable>true</skippable>
<execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts">
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
index c49bb88..79c5a41 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
@@ -259,22 +259,7 @@
</execute-stage>
</group>
- <!-- Invoke hdp-select set all after all components have been stopped but
- before starting new ones. This step must be done here in order to
- ensure that components which rely on other components (like ZKFC on hadoop)
- will start on the correct versions -->
- <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
- <skippable>true</skippable>
- <execute-stage title="Update stack to {{version}}">
- <task xsi:type="execute">
- <script>scripts/ru_set_all.py</script>
- <function>actionexecute</function>
- </task>
- </execute-stage>
- </group>
-
<!-- Now, restart all of the services. -->
-
<group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -514,6 +499,20 @@
</service>
</group>
+ <!--
+ Invoke "hdp-select set all" to change any components we may have missed
+ that are installed on the hosts but not known by Ambari.
+ -->
+ <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+ <skippable>true</skippable>
+ <execute-stage title="Update stack to {{version}}">
+ <task xsi:type="execute">
+ <script>scripts/ru_set_all.py</script>
+ <function>actionexecute</function>
+ </task>
+ </execute-stage>
+ </group>
+
<group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
<skippable>true</skippable>
<execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts">
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py
index 491c686..ba9f920 100644
--- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py
+++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py
@@ -105,7 +105,7 @@ class TestFalconClient(RMFTestCase):
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
- def test_pre_rolling_restart(self):
+ def test_pre_upgrade_restart(self):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
@@ -122,7 +122,7 @@ class TestFalconClient(RMFTestCase):
self.assertNoMoreResources()
@patch("resource_management.core.shell.call")
- def test_pre_rolling_restart_23(self, call_mock):
+ def test_pre_upgrade_restart_23(self, call_mock):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
index be038ab..2219de1 100644
--- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
+++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
@@ -489,7 +489,7 @@ class TestFalconServer(RMFTestCase):
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
@patch('os.path.isfile', new=MagicMock(return_value=True))
- def test_pre_rolling_restart(self):
+ def test_pre_upgrade_restart(self):
config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/falcon-upgrade.json"
with open(config_file, "r") as f:
@@ -511,7 +511,7 @@ class TestFalconServer(RMFTestCase):
@patch.object(tarfile, 'open')
@patch.object(shutil, 'rmtree')
@patch("resource_management.core.shell.call")
- def test_pre_rolling_restart_23(self, tarfile_open_mock, rmtree_mock, call_mock):
+ def test_pre_upgrade_restart_23(self, tarfile_open_mock, rmtree_mock, call_mock):
config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/falcon-upgrade.json"
with open(config_file, "r") as f:
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
index 6f72038..4bf8e9d 100644
--- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
+++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
@@ -226,7 +226,7 @@ class TestKnoxGateway(RMFTestCase):
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
@patch("os.path.isdir")
- def test_pre_rolling_restart(self, isdir_mock):
+ def test_pre_upgrade_restart(self, isdir_mock):
isdir_mock.return_value = True
config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
with open(config_file, "r") as f:
@@ -236,7 +236,7 @@ class TestKnoxGateway(RMFTestCase):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
classname = "KnoxGateway",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -262,7 +262,7 @@ class TestKnoxGateway(RMFTestCase):
@patch("os.path.exists")
@patch("os.path.isdir")
@patch("resource_management.core.shell.call")
- def test_pre_rolling_restart_to_hdp_2300(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+ def test_pre_upgrade_restart_to_hdp_2300(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
"""
In HDP 2.3.0.0, Knox was using a data dir of /var/lib/knox/data
"""
@@ -278,7 +278,7 @@ class TestKnoxGateway(RMFTestCase):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
classname = "KnoxGateway",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
@@ -330,7 +330,7 @@ class TestKnoxGateway(RMFTestCase):
@patch("os.path.exists")
@patch("os.path.isdir")
@patch("resource_management.core.shell.call")
- def test_pre_rolling_restart_from_hdp_2300_to_2320(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+ def test_pre_upgrade_restart_from_hdp_2300_to_2320(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
"""
In RU from HDP 2.3.0.0 to 2.3.2.0, should backup the data dir used by the source version, which
is /var/lib/knox/data
@@ -350,7 +350,7 @@ class TestKnoxGateway(RMFTestCase):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
classname = "KnoxGateway",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
@@ -402,7 +402,7 @@ class TestKnoxGateway(RMFTestCase):
@patch("os.path.exists")
@patch("os.path.isdir")
@patch("resource_management.core.shell.call")
- def test_pre_rolling_restart_from_hdp_2320(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+ def test_pre_upgrade_restart_from_hdp_2320(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
"""
In RU from HDP 2.3.2 to anything higher, should backup the data dir used by the source version, which
is /var/lib/knox/data_${source_version}
@@ -422,7 +422,7 @@ class TestKnoxGateway(RMFTestCase):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
classname = "KnoxGateway",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ed0a03a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_client.py b/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_client.py
index 34dda9a..57d269c 100644
--- a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_client.py
+++ b/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_client.py
@@ -62,7 +62,7 @@ class TestMahoutClient(RMFTestCase):
self.assertNoMoreResources()
- def test_pre_rolling_restart(self):
+ def test_pre_upgrade_restart(self):
config_file = self.get_src_folder() + "/test/python/stacks/2.2/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
@@ -79,7 +79,7 @@ class TestMahoutClient(RMFTestCase):
self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'mahout-client', '2.2.1.0-3242'), sudo=True)
self.assertNoMoreResources()
- def test_pre_rolling_restart_23(self):
+ def test_pre_upgrade_restart_23(self):
config_file = self.get_src_folder() + "/test/python/stacks/2.2/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)