You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/06/30 23:15:46 UTC
ambari git commit: AMBARI-12195. HiveServer2 should pass correct
version to the deregister command during RU (srimanth)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 58fdd3715 -> efa4d6d20
AMBARI-12195. HiveServer2 should pass correct version to the deregister command during RU (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/efa4d6d2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/efa4d6d2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/efa4d6d2
Branch: refs/heads/branch-2.1
Commit: efa4d6d20bae61129f00c01469ebfc2781943c4d
Parents: 58fdd37
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Tue Jun 30 08:46:08 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Tue Jun 30 13:34:52 2015 -0700
----------------------------------------------------------------------
.../libraries/functions/hdp_select.py | 5 +--
.../package/scripts/hive_server_upgrade.py | 38 ++++++++++++++------
.../0.12.0.2.0/package/scripts/params_linux.py | 3 ++
.../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 2 +-
.../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 2 +-
.../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 3 +-
.../stacks/2.0.6/HIVE/test_hive_server.py | 19 +++++++---
7 files changed, 51 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/efa4d6d2/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
index cfdbe13..5fdc35d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
@@ -148,7 +148,7 @@ def get_role_component_current_hdp_version():
return current_hdp_version
-def get_hadoop_dir(target, force_latest_on_upgrade=False):
+def get_hadoop_dir(target, force_latest_on_upgrade=False, upgrade_stack_only=False):
"""
Return the hadoop shared directory in the following override order
1. Use default for 2.1 and lower
@@ -159,6 +159,7 @@ def get_hadoop_dir(target, force_latest_on_upgrade=False):
:target: the target directory
:force_latest_on_upgrade: if True, then this will return the "current" directory
without the HDP version built into the path, such as /usr/hdp/current/hadoop-client
+ :upgrade_stack_only: if True, provides upgrade stack target if present and not current
"""
if not target in HADOOP_DIR_DEFAULTS:
@@ -184,7 +185,7 @@ def get_hadoop_dir(target, force_latest_on_upgrade=False):
# determine if hdp-select has been run and if not, then use the current
# hdp version until this component is upgraded
current_hdp_version = get_role_component_current_hdp_version()
- if current_hdp_version is not None and stack_version != current_hdp_version:
+ if current_hdp_version is not None and stack_version != current_hdp_version and not upgrade_stack_only:
stack_version = current_hdp_version
if target == "home":
http://git-wip-us.apache.org/repos/asf/ambari/blob/efa4d6d2/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py
index d676304..7533dc9 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py
@@ -24,6 +24,9 @@ from resource_management.core.exceptions import Fail
from resource_management.core.resources.system import Execute
from resource_management.core import shell
from resource_management.libraries.functions import format
+from resource_management.libraries.functions import hdp_select
+from resource_management.libraries.functions.version import format_hdp_stack_version
+from resource_management.libraries.functions.version import compare_versions
def pre_upgrade_deregister():
@@ -56,36 +59,51 @@ def pre_upgrade_deregister():
hive_server_conf_dir = "/etc/hive/conf.server"
# deregister
+ hive_execute_path = params.execute_path
+ # If upgrading, the upgrade-target hive binary should be used to call the --deregister command.
+ # If downgrading, the downgrade-source hive binary should be used to call the --deregister command.
+ if "upgrade" == params.upgrade_direction:
+ # hive_bin
+ upgrade_target_version = format_hdp_stack_version(params.version)
+ if upgrade_target_version and compare_versions(upgrade_target_version, "2.2") >= 0:
+ upgrade_target_hive_bin = format('/usr/hdp/{version}/hive/bin')
+ if (os.pathsep + params.hive_bin) in hive_execute_path:
+ hive_execute_path = hive_execute_path.replace(os.pathsep + params.hive_bin, os.pathsep + upgrade_target_hive_bin)
+ # hadoop_bin_dir
+ upgrade_target_hadoop_bin = hdp_select.get_hadoop_dir("bin", upgrade_stack_only=True)
+ upgrade_source_hadoop_bin = params.hadoop_bin_dir
+ if upgrade_target_hadoop_bin and len(upgrade_target_hadoop_bin) > 0 and (os.pathsep + upgrade_source_hadoop_bin) in hive_execute_path:
+ hive_execute_path = hive_execute_path.replace(os.pathsep + upgrade_source_hadoop_bin, os.pathsep + upgrade_target_hadoop_bin)
+
command = format('hive --config {hive_server_conf_dir} --service hiveserver2 --deregister ' + current_hiveserver_version)
- Execute(command, user=params.hive_user, path=params.execute_path, tries=1 )
+ Execute(command, user=params.hive_user, path=hive_execute_path, tries=1 )
def _get_current_hiveserver_version():
"""
- Runs an "hdp-select status hive-server2" check and parses the result in order
+ Runs "hive --version" and parses the result in order
to obtain the current version of hive.
- :return: the hiveserver2 version, such as "hdp-select status hive-server2"
+ :return: the hiveserver2 version, returned by "hive --version"
"""
import params
try:
- command = 'hdp-select status hive-server2'
- return_code, hdp_output = shell.call(command, user=params.hive_user)
+ command = 'hive --version'
+ return_code, hdp_output = shell.call(command, user=params.hive_user, path=params.execute_path)
except Exception, e:
Logger.error(str(e))
- raise Fail('Unable to execute hdp-select command to retrieve the hiveserver2 version.')
+ raise Fail('Unable to execute hive --version command to retrieve the hiveserver2 version.')
if return_code != 0:
raise Fail('Unable to determine the current HiveServer2 version because of a non-zero return code of {0}'.format(str(return_code)))
- # strip "hive-server2 - " off of result and test the version
- current_hive_server_version = re.sub('hive-server2 - ', '', hdp_output)
- match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', current_hive_server_version)
+ match = re.search('^(Hive) ([0-9]+.[0-9]+.\S+)', hdp_output, re.MULTILINE)
if match:
+ current_hive_server_version = match.group(2)
return current_hive_server_version
else:
- raise Fail('The extracted hiveserver2 version "{0}" does not matching any known pattern'.format(current_hive_server_version))
+ raise Fail('The extracted hiveserver2 version "{0}" does not matching any known pattern'.format(hdp_output))
http://git-wip-us.apache.org/repos/asf/ambari/blob/efa4d6d2/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index b80ecf4..0a3f32b 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -60,6 +60,9 @@ version = default("/commandParams/version", None)
# current host stack version
current_version = default("/hostLevelParams/current_version", None)
+# Upgrade direction
+upgrade_direction = default("/commandParams/upgrade_direction", None)
+
hadoop_bin_dir = "/usr/bin"
hadoop_home = '/usr'
hive_bin = '/usr/lib/hive/bin'
http://git-wip-us.apache.org/repos/asf/ambari/blob/efa4d6d2/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
index 3a5033d..1760936 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
@@ -22,7 +22,7 @@
<order>
- <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Backups">
+ <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Upgrade">
<direction>UPGRADE</direction>
<execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HDFS">
<task xsi:type="execute" hosts="master">
http://git-wip-us.apache.org/repos/asf/ambari/blob/efa4d6d2/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index d4d4266..90706ba 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -21,7 +21,7 @@
<target>2.3.*.*</target>
<target-stack>HDP-2.3</target-stack>
<order>
- <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Backups">
+ <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Upgrade">
<direction>UPGRADE</direction>
<execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HDFS">
http://git-wip-us.apache.org/repos/asf/ambari/blob/efa4d6d2/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
index ad785e9..f489bb0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
@@ -21,8 +21,7 @@
<target>2.3.*.*</target>
<order>
-
- <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Backups">
+ <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Upgrade">
<direction>UPGRADE</direction>
<execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HDFS">
<task xsi:type="execute" hosts="master">
http://git-wip-us.apache.org/repos/asf/ambari/blob/efa4d6d2/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index ae03d5a..914b2f1 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -678,7 +678,12 @@ class TestHiveServer(RMFTestCase):
@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
def test_stop_during_upgrade(self, copy_to_hdfs_mock):
- call_side_effects = [(0, "hive-server2 - 2.2.0.0-2041"), (0, "hive-server2 - 2.2.0.0-2041")] * 4
+ hiveServerVersionOutput = """WARNING: Use "yarn jar" to launch YARN applications.
+Hive 1.2.1.2.3.0.0-2434
+Subversion git://ip-10-0-0-90.ec2.internal/grid/0/jenkins/workspace/HDP-dal-centos6/bigtop/build/hive/rpm/BUILD/hive-1.2.1.2.3.0.0 -r a77a00ae765a73b2957337e96ed5a0dbb2e60dfb
+Compiled by jenkins on Sat Jun 20 11:50:41 EDT 2015
+From source with checksum 150f554beae04f76f814f59549dead8b"""
+ call_side_effects = [(0, hiveServerVersionOutput), (0, hiveServerVersionOutput)] * 4
copy_to_hdfs_mock.return_value = True
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
@@ -688,7 +693,7 @@ class TestHiveServer(RMFTestCase):
call_mocks = call_side_effects
)
- self.assertResourceCalled('Execute', 'hive --config /usr/hdp/current/hive-server2/conf/conf.server --service hiveserver2 --deregister 2.2.0.0-2041',
+ self.assertResourceCalled('Execute', 'hive --config /usr/hdp/current/hive-server2/conf/conf.server --service hiveserver2 --deregister 1.2.1.2.3.0.0-2434',
path=['/bin:/usr/hdp/current/hive-server2/bin:/usr/hdp/current/hadoop-client/bin'],
tries=1, user='hive')
@@ -697,8 +702,12 @@ class TestHiveServer(RMFTestCase):
@patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
@patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=True))
def test_stop_during_upgrade_with_default_conf_server(self, copy_to_hdfs_mock):
-
- call_side_effects = [(0, "hive-server2 - 2.2.0.0-2041"), (0, "hive-server2 - 2.2.0.0-2041")] * 4
+ hiveServerVersionOutput = """WARNING: Use "yarn jar" to launch YARN applications.
+Hive 1.2.1.2.3.0.0-2434
+Subversion git://ip-10-0-0-90.ec2.internal/grid/0/jenkins/workspace/HDP-dal-centos6/bigtop/build/hive/rpm/BUILD/hive-1.2.1.2.3.0.0 -r a77a00ae765a73b2957337e96ed5a0dbb2e60dfb
+Compiled by jenkins on Sat Jun 20 11:50:41 EDT 2015
+From source with checksum 150f554beae04f76f814f59549dead8b"""
+ call_side_effects = [(0, hiveServerVersionOutput), (0, hiveServerVersionOutput)] * 4
copy_to_hdfs_mock.return_value = True
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
@@ -708,7 +717,7 @@ class TestHiveServer(RMFTestCase):
call_mocks = call_side_effects
)
- self.assertResourceCalled('Execute', 'hive --config /etc/hive/conf.server --service hiveserver2 --deregister 2.2.0.0-2041',
+ self.assertResourceCalled('Execute', 'hive --config /etc/hive/conf.server --service hiveserver2 --deregister 1.2.1.2.3.0.0-2434',
path=['/bin:/usr/hdp/current/hive-server2/bin:/usr/hdp/current/hadoop-client/bin'],
tries=1, user='hive')