You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/09/21 22:40:27 UTC
ambari git commit: AMBARI-13164. RU: Knox to use versioned data dir
starting in HDP 2.3.2.0 (alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk 26da70b3f -> f9271abc2
AMBARI-13164. RU: Knox to use versioned data dir starting in HDP 2.3.2.0 (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f9271abc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f9271abc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f9271abc
Branch: refs/heads/trunk
Commit: f9271abc25614adbcfa00599db643a24ac20f0f2
Parents: 26da70b
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Sun Sep 20 22:10:48 2015 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Mon Sep 21 13:39:42 2015 -0700
----------------------------------------------------------------------
.../libraries/script/script.py | 18 +-
.../0.5.0.2.2/package/scripts/params_linux.py | 35 +++-
.../KNOX/0.5.0.2.2/package/scripts/upgrade.py | 30 +++-
.../package/scripts/nodemanager_upgrade.py | 2 +-
.../python/stacks/2.2/KNOX/test_knox_gateway.py | 164 ++++++++++++++++++-
5 files changed, 232 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9271abc/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index e0a7877..fc0428d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -308,12 +308,26 @@ class Script(object):
return format_hdp_stack_version(stack_version_unformatted)
@staticmethod
+ def is_hdp_stack_greater(formatted_hdp_stack_version, compare_to_version):
+ """
+ Gets whether the provided formatted_hdp_stack_version (normalized)
+ is greater than the specified stack version
+ :param formatted_hdp_stack_version: the version of stack to compare
+ :param compare_to_version: the version of stack to compare to
+ :return: True if the command's stack is greater than the specified version
+ """
+ if formatted_hdp_stack_version is None or formatted_hdp_stack_version == "":
+ return False
+
+ return compare_versions(formatted_hdp_stack_version, compare_to_version) > 0
+
+ @staticmethod
def is_hdp_stack_greater_or_equal(compare_to_version):
"""
Gets whether the hostLevelParams/stack_version, after being normalized,
is greater than or equal to the specified stack version
:param compare_to_version: the version to compare to
- :return: True if the command's stack is greater than the specified version
+ :return: True if the command's stack is greater than or equal the specified version
"""
return Script.is_hdp_stack_greater_or_equal_to(Script.get_hdp_stack_version(), compare_to_version)
@@ -324,7 +338,7 @@ class Script(object):
is greater than or equal to the specified stack version
:param formatted_hdp_stack_version: the version of stack to compare
:param compare_to_version: the version of stack to compare to
- :return: True if the command's stack is greater than the specified version
+ :return: True if the command's stack is greater than or equal to the specified version
"""
if formatted_hdp_stack_version is None or formatted_hdp_stack_version == "":
return False
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9271abc/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
index 990fc87..5d4ff69 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
@@ -18,7 +18,8 @@ limitations under the License.
Ambari Agent
"""
-from resource_management import *
+from resource_management.core.logger import Logger
+
import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
from resource_management.libraries.functions import format
from resource_management.libraries.functions.version import format_hdp_stack_version
@@ -35,9 +36,37 @@ tmp_dir = Script.get_tmp_dir()
stack_name = default("/hostLevelParams/stack_name", None)
upgrade_direction = default("/commandParams/upgrade_direction", None)
version = default("/commandParams/version", None)
+# E.g., 2.3.2.0
+version_formatted = format_hdp_stack_version(version)
+
+# E.g., 2.3
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
+
+# This is the version whose state is CURRENT. During an RU, this is the source version.
+# DO NOT format it since we need the build number too.
+upgrade_from_version = default("/hostLevelParams/current_version", None)
+
+# server configurations
+# Default value used in HDP 2.3.0.0 and earlier.
+
+knox_data_dir = '/var/lib/knox/data'
+
+# Important, it has to be strictly greater than 2.3.0.0!!!
+if stack_name and stack_name.upper() == "HDP":
+ Logger.info(format("HDP version to use is {version_formatted}"))
+ if Script.is_hdp_stack_greater(version_formatted, "2.3.0.0"):
+ # This is the current version. In the case of a Rolling Upgrade, it will be the newer version.
+ # In the case of a Downgrade, it will be the version downgrading to.
+ # This is always going to be a symlink to /var/lib/knox/data_${version}
+ knox_data_dir = format('/usr/hdp/{version}/knox/data')
+ Logger.info(format("Detected HDP with stack version {version}, will use knox_data_dir = {knox_data_dir}"))
+
+
+knox_logs_dir = '/var/log/knox'
-knox_master_secret_path = '/var/lib/knox/data/security/master'
-knox_cert_store_path = '/var/lib/knox/data/security/keystores/gateway.jks'
+knox_master_secret_path = format('{knox_data_dir}/security/master')
+knox_cert_store_path = format('{knox_data_dir}/security/keystores/gateway.jks')
knox_user = default("/configurations/knox-env/knox_user", "knox")
# server configurations
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9271abc/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
index 2c805fa..a9bb9e9 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
@@ -25,6 +25,10 @@ import tempfile
from resource_management.core.logger import Logger
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions import tar_archive
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import Direction
+from resource_management.libraries.functions.version import compare_versions,format_hdp_stack_version
+
BACKUP_TEMP_DIR = "knox-upgrade-backup"
BACKUP_DATA_ARCHIVE = "knox-data-backup.tar"
@@ -36,7 +40,7 @@ def backup_data():
:return: Returns the path to the absolute backup directory.
"""
Logger.info('Backing up Knox data directory before upgrade...')
- directoryMappings = _get_directory_mappings()
+ directoryMappings = _get_directory_mappings_during_upgrade()
absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)
if not os.path.isdir(absolute_backup_dir):
@@ -58,7 +62,7 @@ def backup_data():
return absolute_backup_dir
-def _get_directory_mappings():
+def _get_directory_mappings_during_upgrade():
"""
Gets a dictionary of directory to archive name that represents the
directories that need to be backed up and their output tarball archive targets
@@ -66,6 +70,24 @@ def _get_directory_mappings():
"""
import params
- return { params.knox_data_dir : BACKUP_DATA_ARCHIVE,
- params.knox_conf_dir + "/": BACKUP_CONF_ARCHIVE} # the trailing "/" is important here so as to not include the "conf" folder itself
+ # Must be performing an Upgrade
+ if params.upgrade_direction is None or params.upgrade_direction != Direction.UPGRADE or \
+ params.upgrade_from_version is None or params.upgrade_from_version == "":
+ Logger.error("Function _get_directory_mappings_during_upgrade() can only be called during a Stack Upgrade in direction UPGRADE.")
+ return {}
+
+ # By default, use this for all stacks.
+ knox_data_dir = '/var/lib/knox/data'
+
+ if params.stack_name and params.stack_name.upper() == "HDP" and \
+ compare_versions(format_hdp_stack_version(params.upgrade_from_version), "2.3.0.0") > 0:
+ # Use the version that is being upgraded from.
+ knox_data_dir = format('/usr/hdp/{upgrade_from_version}/knox/data')
+
+
+ directories = {knox_data_dir: BACKUP_DATA_ARCHIVE,
+ params.knox_conf_dir + "/": BACKUP_CONF_ARCHIVE} # the trailing "/" is important here so as to not include the "conf" folder itself
+
+ Logger.info(format("Knox directories to backup:\n{directories}"))
+ return directories
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9271abc/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
index 9ac2d63..01f8349 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
@@ -41,7 +41,7 @@ def post_upgrade_check():
_check_nodemanager_startup()
-@retry(times=12, sleep_time=10, err_class=Fail)
+@retry(times=30, sleep_time=10, err_class=Fail)
def _check_nodemanager_startup():
'''
Checks that a NodeManager is in a RUNNING state in the cluster via
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9271abc/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
index 3ae42a0..83bba1f 100644
--- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
+++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
@@ -245,7 +245,7 @@ class TestKnoxGateway(RMFTestCase):
self.assertResourceCalled('Execute', ('tar',
'-zcvhf',
'/tmp/knox-upgrade-backup/knox-data-backup.tar',
- '/usr/hdp/current/knox-server/data/'),
+ '/var/lib/knox/data'),
sudo = True,
)
self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', '2.2.1.0-3242'),
@@ -257,12 +257,15 @@ class TestKnoxGateway(RMFTestCase):
@patch("os.path.exists")
@patch("os.path.isdir")
@patch("resource_management.core.shell.call")
- def test_pre_rolling_restart_23(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+ def test_pre_rolling_restart_to_hdp_2300(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+ """
+ In HDP 2.3.0.0, Knox was using a data dir of /var/lib/knox/data
+ """
isdir_mock.return_value = True
config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
with open(config_file, "r") as f:
json_content = json.load(f)
- version = '2.3.0.0-1234'
+ version = "2.3.0.0-1234"
json_content['commandParams']['version'] = version
path_exists_mock.return_value = True
@@ -286,10 +289,10 @@ class TestKnoxGateway(RMFTestCase):
self.assertResourceCalled('Execute', ('tar',
'-zcvhf',
'/tmp/knox-upgrade-backup/knox-data-backup.tar',
- '/usr/hdp/current/knox-server/data/'),
+ '/var/lib/knox/data'),
sudo = True,
)
- self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', '2.3.0.0-1234'),
+ self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', version),
sudo = True,
)
self.assertResourceCalled('Execute', ('cp',
@@ -312,12 +315,159 @@ class TestKnoxGateway(RMFTestCase):
self.assertEquals(1, mocks_dict['call'].call_count)
self.assertEquals(1, mocks_dict['checked_call'].call_count)
self.assertEquals(
- ('conf-select', 'set-conf-dir', '--package', 'knox', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'),
+ ('conf-select', 'set-conf-dir', '--package', 'knox', '--stack-version', version, '--conf-version', '0'),
mocks_dict['checked_call'].call_args_list[0][0][0])
self.assertEquals(
- ('conf-select', 'create-conf-dir', '--package', 'knox', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'),
+ ('conf-select', 'create-conf-dir', '--package', 'knox', '--stack-version', version, '--conf-version', '0'),
mocks_dict['call'].call_args_list[0][0][0])
+ @patch("os.remove")
+ @patch("os.path.exists")
+ @patch("os.path.isdir")
+ @patch("resource_management.core.shell.call")
+ def test_pre_rolling_restart_from_hdp_2300_to_2320(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+ """
+ In RU from HDP 2.3.0.0 to 2.3.2.0, should backup the data dir used by the source version, which
+ is /var/lib/knox/data
+ """
+ isdir_mock.return_value = True
+ config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
+ with open(config_file, "r") as f:
+ json_content = json.load(f)
+ source_version = "2.3.0.0-1234"
+ version = "2.3.2.0-5678"
+ # This is an RU from 2.3.0.0 to 2.3.2.0
+ json_content['commandParams']['version'] = version
+ json_content['hostLevelParams']['current_version'] = source_version
+
+ path_exists_mock.return_value = True
+ mocks_dict = {}
+
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
+ classname = "KnoxGateway",
+ command = "pre_rolling_restart",
+ config_dict = json_content,
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES,
+ call_mocks = [(0, None), (0, None)],
+ mocks_dict = mocks_dict)
+
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+ '/usr/hdp/current/knox-server/conf/'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/knox-upgrade-backup/knox-data-backup.tar',
+ '/var/lib/knox/data'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', version),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('cp',
+ '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+ '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-xvf',
+ '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+ '-C',
+ '/usr/hdp/current/knox-server/conf/'),
+ sudo = True,
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar',
+ action = ['delete'],
+ )
+ self.assertNoMoreResources()
+
+ self.assertEquals(1, mocks_dict['call'].call_count)
+ self.assertEquals(1, mocks_dict['checked_call'].call_count)
+ self.assertEquals(
+ ('conf-select', 'set-conf-dir', '--package', 'knox', '--stack-version', version, '--conf-version', '0'),
+ mocks_dict['checked_call'].call_args_list[0][0][0])
+ self.assertEquals(
+ ('conf-select', 'create-conf-dir', '--package', 'knox', '--stack-version', version, '--conf-version', '0'),
+ mocks_dict['call'].call_args_list[0][0][0])
+
+ @patch("os.remove")
+ @patch("os.path.exists")
+ @patch("os.path.isdir")
+ @patch("resource_management.core.shell.call")
+ def test_pre_rolling_restart_from_hdp_2320(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+ """
+ In RU from HDP 2.3.2 to anything higher, should backup the data dir used by the source version, which
+ is /var/lib/knox/data_${source_version}
+ """
+ isdir_mock.return_value = True
+ config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
+ with open(config_file, "r") as f:
+ json_content = json.load(f)
+ source_version = "2.3.2.0-1000"
+ version = "2.3.2.0-1001"
+ # This is an RU from 2.3.2.0 to 2.3.2.1
+ json_content['commandParams']['version'] = version
+ json_content['hostLevelParams']['current_version'] = source_version
+
+ path_exists_mock.return_value = True
+ mocks_dict = {}
+
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
+ classname = "KnoxGateway",
+ command = "pre_rolling_restart",
+ config_dict = json_content,
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES,
+ call_mocks = [(0, None), (0, None)],
+ mocks_dict = mocks_dict)
+
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+ '/usr/hdp/current/knox-server/conf/'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-zcvhf',
+ '/tmp/knox-upgrade-backup/knox-data-backup.tar',
+ "/usr/hdp/%s/knox/data" % source_version),
+ sudo = True,
+ )
+
+ '''
+ self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', version),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('cp',
+ '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+ '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar'),
+ sudo = True,
+ )
+ self.assertResourceCalled('Execute', ('tar',
+ '-xvf',
+ '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+ '-C',
+ '/usr/hdp/current/knox-server/conf/'),
+ sudo = True,
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar',
+ action = ['delete'],
+ )
+ self.assertNoMoreResources()
+
+ self.assertEquals(1, mocks_dict['call'].call_count)
+ self.assertEquals(1, mocks_dict['checked_call'].call_count)
+ self.assertEquals(
+ ('conf-select', 'set-conf-dir', '--package', 'knox', '--stack-version', version, '--conf-version', '0'),
+ mocks_dict['checked_call'].call_args_list[0][0][0])
+ self.assertEquals(
+ ('conf-select', 'create-conf-dir', '--package', 'knox', '--stack-version', version, '--conf-version', '0'),
+ mocks_dict['call'].call_args_list[0][0][0])
+ '''
+
@patch("os.path.islink")
def test_start_default(self, islink_mock):