You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2015/11/25 20:36:55 UTC
ambari git commit: AMBARI-14073 - Hive Metastore Downgrade Fails
Because Of upgradeSchema Invocation (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/trunk 9a05c0591 -> ebba6caec
AMBARI-14073 - Hive Metastore Downgrade Fails Because Of upgradeSchema Invocation (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ebba6cae
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ebba6cae
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ebba6cae
Branch: refs/heads/trunk
Commit: ebba6caec221b7c915cdd9c0717c1b4d48075420
Parents: 9a05c05
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Nov 25 12:26:58 2015 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Wed Nov 25 14:36:32 2015 -0500
----------------------------------------------------------------------
.../package/scripts/hive_metastore.py | 22 ++++---
.../0.12.0.2.0/package/scripts/params_linux.py | 5 +-
.../stacks/2.1/HIVE/test_hive_metastore.py | 63 +++++++++++++++-----
3 files changed, 65 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ebba6cae/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
index 084494d..d26e36f 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
@@ -24,6 +24,7 @@ from resource_management.core.resources.system import Execute, Directory
from resource_management.libraries.script import Script
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import hdp_select
+from resource_management.libraries.functions.constants import Direction
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.version import format_hdp_stack_version
from resource_management.libraries.functions.version import compare_versions
@@ -99,16 +100,12 @@ class HiveMetastoreDefault(HiveMetastore):
Logger.info("Executing Metastore Stack Upgrade pre-restart")
import params
- if Script.is_hdp_stack_greater_or_equal("2.3"):
- # disable schema creation on upgrade
- params.init_metastore_schema = False
-
env.set_params(params)
- if Script.is_hdp_stack_greater_or_equal("2.3"):
- # ensure that configurations are written out before trying to upgrade the schema
- # since the schematool needs configs and doesn't know how to use the hive conf override
- self.configure(env)
+ is_stack_hdp_23 = Script.is_hdp_stack_greater_or_equal("2.3")
+ is_upgrade = params.upgrade_direction == Direction.UPGRADE
+
+ if is_stack_hdp_23 and is_upgrade:
self.upgrade_schema(env)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
@@ -176,11 +173,20 @@ class HiveMetastoreDefault(HiveMetastore):
The metastore schema upgrade requires a database driver library for most
databases. During an upgrade, it's possible that the library is not present,
so this will also attempt to copy/download the appropriate driver.
+
+ This function will also ensure that configurations are written out to disk before running
+ since the new configs will most likely not yet exist on an upgrade.
+
+ Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades.
"""
Logger.info("Upgrading Hive Metastore Schema")
import params
env.set_params(params)
+ # ensure that configurations are written out before trying to upgrade the schema
+ # since the schematool needs configs and doesn't know how to use the hive conf override
+ self.configure(env)
+
if params.security_enabled:
kinit_command=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ")
Execute(kinit_command,user=params.smokeuser)
http://git-wip-us.apache.org/repos/asf/ambari/blob/ebba6cae/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 972d532..f360651 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -318,8 +318,9 @@ mysql_host = config['clusterHostInfo']['hive_mysql_host']
mysql_adduser_path = format("{tmp_dir}/addMysqlUser.sh")
mysql_deluser_path = format("{tmp_dir}/removeMysqlUser.sh")
-######## Metastore Schema
-init_metastore_schema = True
+#### Metastore
+# initialize the schema only if not in an upgrade/downgrade
+init_metastore_schema = upgrade_direction is None
########## HCAT
hcat_dbroot = hcat_lib
http://git-wip-us.apache.org/repos/asf/ambari/blob/ebba6cae/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
index bdc0fe2..be3a956 100644
--- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
@@ -23,6 +23,7 @@ import os
from mock.mock import MagicMock, call, patch
from stacks.utils.RMFTestCase import *
+from resource_management.libraries.functions.constants import Direction
from resource_management.libraries.script.script import Script
@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
@@ -534,31 +535,60 @@ class TestHiveMetastore(RMFTestCase):
('ambari-python-wrap', '/usr/bin/conf-select', 'create-conf-dir', '--package', 'hive', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'),
mocks_dict['call'].call_args_list[0][0][0])
- @patch.object(Script, "is_hdp_stack_greater_or_equal")
- def test_pre_upgrade_restart_ims(self, is_hdp_stack_greater_or_equal_mock):
+ def test_pre_upgrade_restart_ims(self):
"""
Tests the state of the init_metastore_schema property on update
"""
- is_hdp_stack_greater_or_equal_mock.side_effect = [False, False, False, False, False, False, True, False, False, False, False, False, False, False]
config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
- stack_version = '2.3.0.0-0'
- version = '2.3.0.0-1234'
+ # first try it with a normal, non-upgrade
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
+ classname = "HiveMetastore",
+ command = "pre_upgrade_restart",
+ config_dict = json_content,
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES,
+ call_mocks = [(0, None, ''), (0, None)])
- json_content['commandParams']['version'] = version
- json_content['hostLevelParams']['stack_version'] = stack_version
+ self.assertEquals(True, RMFTestCase.env.config["params"]["init_metastore_schema"])
- mocks_dict = {}
+ self.config_dict = None
+ config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json"
+ with open(config_file, "r") as f:
+ json_content = json.load(f)
+
+ json_content['commandParams']['version'] = '2.3.0.0-1234'
+ json_content['commandParams']['upgrade_direction'] = Direction.UPGRADE
+ json_content['hostLevelParams']['stack_version'] = '2.3.0.0-0'
+
+ # now try it in an upgrade
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
- classname = "HiveMetastore",
- command = "pre_upgrade_restart",
- config_dict = json_content,
- hdp_stack_version = self.STACK_VERSION,
- target = RMFTestCase.TARGET_COMMON_SERVICES,
- call_mocks = [(0, None, ''), (0, None)],
- mocks_dict = mocks_dict)
+ classname = "HiveMetastore",
+ command = "pre_upgrade_restart",
+ config_dict = json_content,
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES,
+ call_mocks = [(0, None, ''), (0, None)])
+
+ self.assertEquals(False, RMFTestCase.env.config["params"]["init_metastore_schema"])
+
+ self.config_dict = None
+ config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json"
+ with open(config_file, "r") as f:
+ json_content = json.load(f)
+
+ json_content['commandParams']['upgrade_direction'] = Direction.DOWNGRADE
+
+ # now try it in a downgrade
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
+ classname = "HiveMetastore",
+ command = "pre_upgrade_restart",
+ config_dict = json_content,
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES,
+ call_mocks = [(0, None, ''), (0, None)])
self.assertEquals(False, RMFTestCase.env.config["params"]["init_metastore_schema"])
@@ -586,9 +616,11 @@ class TestHiveMetastore(RMFTestCase):
# must be HDP 2.3+
version = '2.3.0.0-1234'
json_content['commandParams']['version'] = version
+ json_content['commandParams']['upgrade_direction'] = Direction.UPGRADE
json_content['hostLevelParams']['stack_version'] = "2.3"
json_content['hostLevelParams']['current_version'] = "2.2.7.0-1234"
+
# trigger the code to think it needs to copy the JAR
json_content['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] = "com.mysql.jdbc.Driver"
json_content['configurations']['hive-env']['hive_database'] = "Existing"
@@ -674,6 +706,7 @@ class TestHiveMetastore(RMFTestCase):
# must be HDP 2.3+
version = '2.3.0.0-1234'
json_content['commandParams']['version'] = version
+ json_content['commandParams']['upgrade_direction'] = Direction.UPGRADE
json_content['hostLevelParams']['stack_version'] = "2.3"
# trigger the code to think it needs to copy the JAR