You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2015/10/19 15:54:22 UTC
[36/50] [abbrv] ambari git commit: AMBARI-13459 - Hive Metastore
Upgrade Retry Fails Because Of Driver Copy Issue (jonathanhurley)
AMBARI-13459 - Hive Metastore Upgrade Retry Fails Because Of Driver Copy Issue (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d8d19268
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d8d19268
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d8d19268
Branch: refs/heads/branch-dev-patch-upgrade
Commit: d8d19268a4c7078216d3a1dd3d589d9ff0cf23bb
Parents: 4023dcf
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Fri Oct 16 14:09:26 2015 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Fri Oct 16 16:07:50 2015 -0400
----------------------------------------------------------------------
.../package/scripts/hive_metastore.py | 30 +++++++++++++-------
.../stacks/2.1/HIVE/test_hive_metastore.py | 8 ++++--
2 files changed, 24 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8d19268/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
index 796ec18..8a3833b 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
@@ -33,7 +33,6 @@ from resource_management.libraries.functions.security_commons import get_params_
from resource_management.libraries.functions.security_commons import validate_security_config_properties
from resource_management.libraries.functions.security_commons import FILE_TYPE_XML
from resource_management.core.resources.system import File
-from resource_management.core.shell import as_sudo
from hive import hive
from hive import jdbc_connector
@@ -155,7 +154,7 @@ class HiveMetastoreDefault(HiveMetastore):
"""
Executes the schema upgrade binary. This is its own function because it could
be called as a standalone task from the upgrade pack, but is safe to run it for each
- metastore instance.
+ metastore instance. The schema upgrade on an already upgraded metastore is a NOOP.
The metastore schema upgrade requires a database driver library for most
databases. During an upgrade, it's possible that the library is not present,
@@ -173,28 +172,36 @@ class HiveMetastoreDefault(HiveMetastore):
# present, then download it first
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
target_directory = format("/usr/hdp/{version}/hive/lib")
- if not os.path.exists(params.target):
- # download it
+
+ # normally, the JDBC driver would be referenced by /usr/hdp/current/.../foo.jar
+ # but if hdp-select is called and the restart fails, then this means that current pointer
+ # is now pointing to the upgraded version location; that's bad for the cp command
+ source_jdbc_file = format(params.target.replace("/usr/hdp/current", "/usr/hdp/{current_version}"))
+
+ # download it if it does not exist
+ if not os.path.exists(source_jdbc_file):
jdbc_connector()
+ target_directory_and_filename = os.path.join(target_directory, os.path.basename(source_jdbc_file))
+
if params.sqla_db_used:
target_native_libs_directory = format("{target_directory}/native/lib64")
Execute(format("yes | {sudo} cp {jars_in_hive_lib} {target_directory}"))
- Directory(target_native_libs_directory,
- recursive=True)
+ Directory(target_native_libs_directory, recursive=True)
Execute(format("yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}"))
Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))
else:
- Execute(('cp', params.target, target_directory),
- path=["/bin", "/usr/bin/"], sudo = True)
+ # copy the JDBC driver from the older metastore location to the new location only
+ # if it does not already exist
+ if not os.path.exists(target_directory_and_filename):
+ Execute(('cp', source_jdbc_file, target_directory),
+ path=["/bin", "/usr/bin/"], sudo = True)
- File(os.path.join(target_directory, os.path.basename(params.target)),
- mode = 0644,
- )
+ File(target_directory_and_filename, mode = 0644)
# build the schema tool command
binary = format("/usr/hdp/{version}/hive/bin/schematool")
@@ -215,5 +222,6 @@ class HiveMetastoreDefault(HiveMetastore):
command = format("{binary} -dbType {hive_metastore_db_type} -upgradeSchema")
Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True)
+
if __name__ == "__main__":
HiveMetastore().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8d19268/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
index a51f139..5e9fc1d 100644
--- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
@@ -539,7 +539,7 @@ class TestHiveMetastore(RMFTestCase):
get_hdp_version_mock.return_value = '2.3.0.0-1234'
def side_effect(path):
- if path == "/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar":
+ if path == "/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar":
return True
return False
@@ -554,6 +554,8 @@ class TestHiveMetastore(RMFTestCase):
version = '2.3.0.0-1234'
json_content['commandParams']['version'] = version
json_content['hostLevelParams']['stack_version'] = "2.3"
+ json_content['hostLevelParams']['current_version'] = "2.2.7.0-1234"
+
# trigger the code to think it needs to copy the JAR
json_content['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] = "com.mysql.jdbc.Driver"
@@ -570,7 +572,7 @@ class TestHiveMetastore(RMFTestCase):
mocks_dict = mocks_dict)
self.assertResourceCalled('Execute',
- ('cp', '/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar', '/usr/hdp/2.3.0.0-1234/hive/lib'),
+ ('cp', '/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar', '/usr/hdp/2.3.0.0-1234/hive/lib'),
path = ['/bin', '/usr/bin/'], sudo = True)
self.assertResourceCalled('File', '/usr/hdp/2.3.0.0-1234/hive/lib/mysql-connector-java.jar',
@@ -578,7 +580,7 @@ class TestHiveMetastore(RMFTestCase):
)
self.assertResourceCalled('Execute', "/usr/hdp/2.3.0.0-1234/hive/bin/schematool -dbType mysql -upgradeSchema",
- logoutput = True, environment = {'HIVE_CONF_DIR': '/usr/hdp/current/hive-server2/conf/conf.server'},
+ logoutput = True, environment = {'HIVE_CONF_DIR': '/etc/hive/conf.server'},
tries = 1, user = 'hive')
self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-metastore', version), sudo=True,)