You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2015/10/28 05:56:59 UTC
ambari git commit: AMBARI-13588 - Express Upgrade Packs For Hive
(jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/trunk 9e085dfd0 -> deb782c79
AMBARI-13588 - Express Upgrade Packs For Hive (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/deb782c7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/deb782c7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/deb782c7
Branch: refs/heads/trunk
Commit: deb782c796cf4ee792f1f1d98769583698b89f60
Parents: 9e085df
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Tue Oct 27 16:15:02 2015 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Wed Oct 28 00:56:28 2015 -0400
----------------------------------------------------------------------
.../src/main/python/ambari_commons/constants.py | 9 ++++--
.../libraries/script/script.py | 5 +--
.../0.12.0.2.0/package/scripts/hive_client.py | 4 ++-
.../package/scripts/hive_metastore.py | 28 ++++++++++++-----
.../0.12.0.2.0/package/scripts/hive_server.py | 26 ++++++++++-----
.../0.12.0.2.0/package/scripts/hive_service.py | 9 +++---
.../package/scripts/setup_ranger_hive.py | 5 +--
.../package/scripts/webhcat_server.py | 10 +++---
.../package/scripts/webhcat_service.py | 4 +--
.../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 2 +-
.../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml | 2 +-
.../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 2 +-
.../stacks/HDP/2.3/upgrades/config-upgrade.xml | 2 +-
.../stacks/2.0.6/HIVE/test_hive_client.py | 8 ++---
.../stacks/2.0.6/HIVE/test_hive_metastore.py | 4 ++-
.../stacks/2.0.6/HIVE/test_hive_server.py | 8 ++---
.../stacks/2.0.6/HIVE/test_webhcat_server.py | 8 ++---
.../stacks/2.1/HIVE/test_hive_metastore.py | 33 ++++++++++++++------
.../src/test/python/stacks/utils/RMFTestCase.py | 8 ++++-
19 files changed, 115 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-common/src/main/python/ambari_commons/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/constants.py b/ambari-common/src/main/python/ambari_commons/constants.py
index 99ff679..525e024 100644
--- a/ambari-common/src/main/python/ambari_commons/constants.py
+++ b/ambari-common/src/main/python/ambari_commons/constants.py
@@ -1,6 +1,5 @@
#!/usr/bin/env python
-
-'''
+"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -16,6 +15,10 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-'''
+
+"""
AMBARI_SUDO_BINARY = "ambari-sudo.sh"
+
+UPGRADE_TYPE_ROLLING = "rolling"
+UPGRADE_TYPE_NON_ROLLING = "nonrolling"
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 79bc535..10a9aa6 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -29,6 +29,7 @@ import platform
import inspect
import tarfile
from ambari_commons import OSCheck, OSConst
+from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from resource_management.libraries.resources import XmlConfig
from resource_management.libraries.resources import PropertiesFile
@@ -468,9 +469,9 @@ class Script(object):
upgrade_type = None
if restart_type.lower() == "rolling_upgrade":
- upgrade_type = "rolling"
+ upgrade_type = UPGRADE_TYPE_ROLLING
elif restart_type.lower() == "nonrolling_upgrade":
- upgrade_type = "nonrolling"
+ upgrade_type = UPGRADE_TYPE_NON_ROLLING
is_stack_upgrade = upgrade_type is not None
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py
index 0cacb07..117b456 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py
@@ -50,7 +50,9 @@ class HiveClientDefault(HiveClient):
def get_stack_to_component(self):
return {"HDP": "hadoop-client"}
- def pre_rolling_restart(self, env):
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ Logger.info("Executing Hive client Stack Upgrade pre-restart")
+
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
index 8bdccdc..080225a 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
@@ -48,16 +48,22 @@ class HiveMetastore(Script):
import params
self.install_packages(env, exclude_packages = params.hive_exclude_packages)
- def start(self, env, rolling_restart=False):
+
+ def start(self, env, upgrade_type=None):
import params
env.set_params(params)
- self.configure(env) # FOR SECURITY
- hive_service('metastore', action='start', rolling_restart=rolling_restart)
- def stop(self, env, rolling_restart=False):
+ # writing configurations on start required for securtity
+ self.configure(env)
+
+ hive_service('metastore', action='start', upgrade_type=upgrade_type)
+
+
+ def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
- hive_service('metastore', action='stop')
+ hive_service('metastore', action='stop', upgrade_type=upgrade_type)
+
def configure(self, env):
import params
@@ -78,6 +84,7 @@ class HiveMetastoreDefault(HiveMetastore):
def get_stack_to_component(self):
return {"HDP": "hive-metastore"}
+
def status(self, env):
import status_params
from resource_management.libraries.functions import check_process_status
@@ -87,18 +94,23 @@ class HiveMetastoreDefault(HiveMetastore):
# Recursively check all existing gmetad pid files
check_process_status(pid_file)
- def pre_rolling_restart(self, env):
- Logger.info("Executing Metastore Rolling Upgrade pre-restart")
+
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ Logger.info("Executing Metastore Stack Upgrade pre-restart")
import params
env.set_params(params)
if Script.is_hdp_stack_greater_or_equal("2.3"):
+ # ensure that configurations are written out before trying to upgrade the schema
+ # since the schematool needs configs and doesn't know how to use the hive conf override
+ self.configure(env)
self.upgrade_schema(env)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "hive", params.version)
hdp_select.select("hive-metastore", params.version)
+
def security_status(self, env):
import status_params
env.set_params(status_params)
@@ -160,7 +172,7 @@ class HiveMetastoreDefault(HiveMetastore):
databases. During an upgrade, it's possible that the library is not present,
so this will also attempt to copy/download the appropriate driver.
"""
- Logger.info("Upgrading Hive Metastore")
+ Logger.info("Upgrading Hive Metastore Schema")
import params
env.set_params(params)
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
index d7542e9..96c05ca 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
@@ -36,6 +36,7 @@ if OSCheck.is_windows_family():
from resource_management.libraries.functions.windows_service_utils import check_windows_service_status
from setup_ranger_hive import setup_ranger_hive
from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons.constants import UPGRADE_TYPE_ROLLING
from resource_management.core.logger import Logger
import hive_server_upgrade
@@ -77,26 +78,31 @@ class HiveServerDefault(HiveServer):
def get_stack_to_component(self):
return {"HDP": "hive-server2"}
- def start(self, env, rolling_restart=False):
+ def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
- setup_ranger_hive(rolling_upgrade=rolling_restart)
- hive_service( 'hiveserver2', action = 'start', rolling_restart=rolling_restart)
- if rolling_restart:
+ setup_ranger_hive(upgrade_type=upgrade_type)
+ hive_service('hiveserver2', action = 'start', upgrade_type=upgrade_type)
+
+ # only perform this if upgrading and rolling; a non-rolling upgrade doesn't need
+ # to do this since hive is already down
+ if upgrade_type == UPGRADE_TYPE_ROLLING:
hive_server_upgrade.post_upgrade_deregister()
- def stop(self, env, rolling_restart=False):
+
+ def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
# During rolling upgrade, HiveServer2 should not be stopped before new server is available.
# Once new server is started, old one is stopped by the --deregister command which is
# invoked by the 'hive_server_upgrade.post_upgrade_deregister()' method
- if not rolling_restart:
+ if upgrade_type != UPGRADE_TYPE_ROLLING:
hive_service( 'hiveserver2', action = 'stop' )
+
def status(self, env):
import status_params
env.set_params(status_params)
@@ -105,8 +111,9 @@ class HiveServerDefault(HiveServer):
# Recursively check all existing gmetad pid files
check_process_status(pid_file)
- def pre_rolling_restart(self, env):
- Logger.info("Executing HiveServer2 Rolling Upgrade pre-restart")
+
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ Logger.info("Executing Hive Server Stack Upgrade pre-restart")
import params
env.set_params(params)
@@ -120,14 +127,17 @@ class HiveServerDefault(HiveServer):
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
+
resource_created = copy_to_hdfs(
"tez",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped) or resource_created
+
if resource_created:
params.HdfsResource(None, action="execute")
+
def security_status(self, env):
import status_params
env.set_params(status_params)
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
index ccb4f7d..456c018 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
@@ -21,6 +21,7 @@ limitations under the License.
import os
import time
+from ambari_commons.constants import UPGRADE_TYPE_ROLLING
from resource_management.core.logger import Logger
from resource_management.core import shell
from resource_management.libraries.functions.format import format
@@ -35,7 +36,7 @@ from ambari_commons import OSConst
@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def hive_service(name, action='start', rolling_restart=False):
+def hive_service(name, action='start', upgrade_type=None):
import params
if name == 'metastore':
if action == 'start' or action == 'stop':
@@ -47,7 +48,7 @@ def hive_service(name, action='start', rolling_restart=False):
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def hive_service(name, action='start', rolling_restart=False):
+def hive_service(name, action='start', upgrade_type=None):
import params
@@ -72,10 +73,10 @@ def hive_service(name, action='start', rolling_restart=False):
# upgrading hiveserver2 (rolling_restart) means that there is an existing,
# de-registering hiveserver2; the pid will still exist, but the new
# hiveserver is spinning up on a new port, so the pid will be re-written
- if rolling_restart:
+ if upgrade_type == UPGRADE_TYPE_ROLLING:
process_id_exists_command = None
- if (params.version):
+ if params.version:
import os
hadoop_home = format("/usr/hdp/{version}/hadoop")
hive_bin = os.path.join(params.hive_bin, hive_bin)
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
index 5fdaa70..c17def0 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
@@ -19,7 +19,7 @@ limitations under the License.
"""
from resource_management.core.logger import Logger
-def setup_ranger_hive(rolling_upgrade = False):
+def setup_ranger_hive(upgrade_type = None):
import params
if params.has_ranger_admin:
@@ -30,7 +30,8 @@ def setup_ranger_hive(rolling_upgrade = False):
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
hdp_version = None
- if rolling_upgrade:
+
+ if upgrade_type is not None:
hdp_version = params.version
if params.retryAble:
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py
index 412edc7..f0f31a8 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py
@@ -35,13 +35,13 @@ class WebHCatServer(Script):
import params
self.install_packages(env, exclude_packages=params.hive_exclude_packages)
- def start(self, env, rolling_restart=False):
+ def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
- webhcat_service(action='start', rolling_restart=rolling_restart)
+ webhcat_service(action='start', upgrade_type=upgrade_type)
- def stop(self, env, rolling_restart=False):
+ def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
webhcat_service(action='stop')
@@ -70,8 +70,8 @@ class WebHCatServerDefault(WebHCatServer):
env.set_params(status_params)
check_process_status(status_params.webhcat_pid_file)
- def pre_rolling_restart(self, env):
- Logger.info("Executing WebHCat Rolling Upgrade pre-restart")
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ Logger.info("Executing WebHCat Stack Upgrade pre-restart")
import params
env.set_params(params)
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py
index 46ad109..2553c70 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py
@@ -31,7 +31,7 @@ def webhcat_service(action='start', rolling_restart=False):
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def webhcat_service(action='start', rolling_restart=False):
+def webhcat_service(action='start', upgrade_type=None):
import params
environ = {
@@ -41,7 +41,7 @@ def webhcat_service(action='start', rolling_restart=False):
cmd = format('{webhcat_bin_dir}/webhcat_server.sh')
if action == 'start':
- if rolling_restart and params.version:
+ if upgrade_type is not None and params.version:
environ['HADOOP_HOME'] = format("/usr/hdp/{version}/hadoop")
daemon_cmd = format('cd {hcat_pid_dir} ; {cmd} start')
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
index 0f6b8f1..25fd6ab 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
@@ -246,7 +246,7 @@
</execute-stage>
<execute-stage service="HIVE" component="HIVE_SERVER">
- <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentification"/>
+ <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/>
</execute-stage>
<execute-stage service="HIVE" component="WEBHCAT_SERVER">
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
index b5add07..44413d3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
@@ -381,7 +381,7 @@
</execute-stage>
<execute-stage service="HIVE" component="HIVE_SERVER">
- <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentification"/>
+ <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/>
</execute-stage>
<execute-stage service="HIVE" component="HIVE_SERVER">
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index c091bb7..d3e209e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -610,7 +610,7 @@
<task xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager"/>
- <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentification"/>
+ <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/>
<task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_ranger_policy"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index fe8e1fe..618b60a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -494,7 +494,7 @@
<replace key="hive.security.authorization.manager" find="com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory" replace-with="org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory" />
</definition>
- <definition xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentification" summary="Configuring hive authentication">
+ <definition xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication" summary="Configuring hive authentication">
<type>hive-site</type>
<transfer operation="delete" delete-key="hive.server2.authentication.ldap.url" if-key="hive.server2.authentication" if-type="hive-site" if-value="NONE"/>
<transfer operation="delete" delete-key="hive.server2.authentication.ldap.baseDN" if-key="hive.server2.authentication" if-type="hive-site" if-value="NONE"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
index 97898b8..d600a65 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
@@ -175,7 +175,7 @@ class TestHiveClient(RMFTestCase):
)
self.assertNoMoreResources()
- def test_pre_rolling_restart(self):
+ def test_pre_upgrade_restart(self):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
@@ -183,7 +183,7 @@ class TestHiveClient(RMFTestCase):
json_content['commandParams']['version'] = version
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_client.py",
classname = "HiveClient",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -192,7 +192,7 @@ class TestHiveClient(RMFTestCase):
self.assertNoMoreResources()
@patch("resource_management.core.shell.call")
- def test_pre_rolling_restart_23(self, call_mock):
+ def test_pre_upgrade_restart_23(self, call_mock):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
@@ -202,7 +202,7 @@ class TestHiveClient(RMFTestCase):
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_client.py",
classname = "HiveClient",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
index 7f3efcb..517d71c 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
@@ -364,6 +364,8 @@ class TestHiveMetastore(RMFTestCase):
@patch("resource_management.core.shell.call")
@patch("resource_management.libraries.functions.get_hdp_version")
def test_start_ru(self, call_mock, get_hdp_version_mock):
+ from ambari_commons.constants import UPGRADE_TYPE_ROLLING
+
get_hdp_version_mock.return_value = '2.3.0.0-1234'
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
@@ -380,7 +382,7 @@ class TestHiveMetastore(RMFTestCase):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
classname = "HiveMetastore",
command = "start",
- command_args = [True],
+ command_args = [UPGRADE_TYPE_ROLLING],
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES)
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index 14ed232..51bc3a2 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -890,7 +890,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
@patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=True))
@patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
- def test_pre_rolling_restart(self, copy_to_hdfs_mock):
+ def test_pre_upgrade_restart(self, copy_to_hdfs_mock):
copy_to_hdfs_mock.return_value = True
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
@@ -900,7 +900,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
json_content['commandParams']['version'] = version
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
classname = "HiveServer",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -925,7 +925,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
@patch("resource_management.core.shell.call")
@patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=True))
@patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
- def test_pre_rolling_restart_23(self, copy_to_hdfs_mock, call_mock):
+ def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock, call_mock):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
@@ -936,7 +936,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
classname = "HiveServer",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
index b0b34bc..b9b1ce9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
@@ -343,7 +343,7 @@ class TestWebHCatServer(RMFTestCase):
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
- def test_pre_rolling_restart(self):
+ def test_pre_upgrade_restart(self):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
@@ -351,7 +351,7 @@ class TestWebHCatServer(RMFTestCase):
json_content['commandParams']['version'] = version
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/webhcat_server.py",
classname = "WebHCatServer",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -360,7 +360,7 @@ class TestWebHCatServer(RMFTestCase):
self.assertNoMoreResources()
@patch("resource_management.core.shell.call")
- def test_pre_rolling_restart_23(self, call_mock):
+ def test_pre_upgrade_restart_23(self, call_mock):
import sys
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
@@ -373,7 +373,7 @@ class TestWebHCatServer(RMFTestCase):
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/webhcat_server.py",
classname = "WebHCatServer",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
index 40f0b3b..2800224 100644
--- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
@@ -483,7 +483,7 @@ class TestHiveMetastore(RMFTestCase):
)
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
- def test_pre_rolling_restart(self):
+ def test_pre_upgrade_restart(self):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
@@ -491,7 +491,7 @@ class TestHiveMetastore(RMFTestCase):
json_content['commandParams']['version'] = version
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
classname = "HiveMetastore",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -500,7 +500,7 @@ class TestHiveMetastore(RMFTestCase):
self.assertNoMoreResources()
@patch("resource_management.core.shell.call")
- def test_pre_rolling_restart_23(self, call_mock):
+ def test_pre_upgrade_restart_23(self, call_mock):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
@@ -510,7 +510,7 @@ class TestHiveMetastore(RMFTestCase):
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
classname = "HiveMetastore",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
@@ -535,12 +535,13 @@ class TestHiveMetastore(RMFTestCase):
@patch("resource_management.core.shell.call")
@patch("resource_management.libraries.functions.get_hdp_version")
def test_upgrade_metastore_schema(self, get_hdp_version_mock, call_mock, os_path_exists_mock):
-
get_hdp_version_mock.return_value = '2.3.0.0-1234'
def side_effect(path):
if path == "/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar":
return True
+ if ".j2" in path:
+ return True
return False
os_path_exists_mock.side_effect = side_effect
@@ -556,7 +557,6 @@ class TestHiveMetastore(RMFTestCase):
json_content['hostLevelParams']['stack_version'] = "2.3"
json_content['hostLevelParams']['current_version'] = "2.2.7.0-1234"
-
# trigger the code to think it needs to copy the JAR
json_content['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] = "com.mysql.jdbc.Driver"
json_content['configurations']['hive-env']['hive_database'] = "Existing"
@@ -564,13 +564,17 @@ class TestHiveMetastore(RMFTestCase):
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
classname = "HiveMetastore",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
call_mocks = [(0, None), (0, None)],
mocks_dict = mocks_dict)
+ # we don't care about configure here - the strings are different anyway because this
+ # is an upgrade, so just pop those resources off of the call stack
+ self.pop_resources(21)
+
self.assertResourceCalled('Execute', ('rm', '-f', '/usr/hdp/current/hive-server2/lib/ojdbc6.jar'),
path = ['/bin', '/usr/bin/'],
sudo = True,
@@ -619,7 +623,14 @@ class TestHiveMetastore(RMFTestCase):
get_hdp_version_mock.return_value = '2.3.0.0-1234'
- os_path_exists_mock.return_value = False
+ def side_effect(path):
+ if path == "/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar":
+ return True
+ if ".j2" in path:
+ return True
+ return False
+
+ os_path_exists_mock.side_effect = side_effect
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
@@ -639,13 +650,17 @@ class TestHiveMetastore(RMFTestCase):
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
classname = "HiveMetastore",
- command = "pre_rolling_restart",
+ command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
call_mocks = [(0, None), (0, None)],
mocks_dict = mocks_dict)
+ # we don't care about configure here - the strings are different anyway because this
+ # is an upgrade, so just pop those resources off of the call stack
+ self.pop_resources(25)
+
self.assertResourceCalled('Execute',
('rm', '-f', '/usr/hdp/current/hive-server2/lib/ojdbc6.jar'),
path=["/bin", "/usr/bin/"],
http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
index a186527..ab4eed4 100644
--- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
+++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
@@ -223,7 +223,13 @@ class RMFTestCase(TestCase):
s = self.reindent(s, intendation)
print s
print(self.reindent("self.assertNoMoreResources()", intendation))
-
+
+ def pop_resources(self, count):
+ with patch.object(UnknownConfiguration, '__getattr__', return_value=lambda: "UnknownConfiguration()"):
+ self.assertNotEqual(len(RMFTestCase.env.resource_list), 0, "There was no more resources executed!")
+ for i in range(count):
+ RMFTestCase.env.resource_list.pop(0)
+
def assertResourceCalled(self, resource_type, name, **kwargs):
with patch.object(UnknownConfiguration, '__getattr__', return_value=lambda: "UnknownConfiguration()"):
self.assertNotEqual(len(RMFTestCase.env.resource_list), 0, "There was no more resources executed!")