You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2017/07/28 13:00:03 UTC
ambari git commit: AMBARI-21595. Ambari Server Setup Code Fails
Because of Missing Stack Upgrade (dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-21450 56d2ade2b -> dd8d7109f
AMBARI-21595. Ambari Server Setup Code Fails Because of Missing Stack Upgrade (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dd8d7109
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dd8d7109
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dd8d7109
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: dd8d7109f63f96cb363203be99047a31b415da33
Parents: 56d2ade
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Jul 28 15:59:23 2017 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Jul 28 15:59:23 2017 +0300
----------------------------------------------------------------------
ambari-server/src/main/python/ambari-server.py | 9 +-
.../main/python/ambari_server/setupActions.py | 1 -
.../src/test/python/TestAmbariServer.py | 168 +------------------
.../src/test/python/TestServerUpgrade.py | 36 +---
4 files changed, 12 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd8d7109/ambari-server/src/main/python/ambari-server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py
index b25cd16..60c6005 100755
--- a/ambari-server/src/main/python/ambari-server.py
+++ b/ambari-server/src/main/python/ambari-server.py
@@ -37,7 +37,7 @@ from ambari_server.dbConfiguration import DATABASE_NAMES, LINUX_DBMS_KEYS_LIST
from ambari_server.serverConfiguration import configDefaults, get_ambari_properties, PID_NAME
from ambari_server.serverUtils import is_server_runing, refresh_stack_hash, wait_for_server_to_stop
from ambari_server.serverSetup import reset, setup, setup_jce_policy
-from ambari_server.serverUpgrade import upgrade, upgrade_stack, set_current
+from ambari_server.serverUpgrade import upgrade, set_current
from ambari_server.setupHttps import setup_https, setup_truststore
from ambari_server.setupMpacks import install_mpack, uninstall_mpack, upgrade_mpack, STACK_DEFINITIONS_RESOURCE_NAME, \
SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME
@@ -49,9 +49,9 @@ from ambari_server.enableStack import enable_stack_version
from ambari_server.setupActions import BACKUP_ACTION, LDAP_SETUP_ACTION, LDAP_SYNC_ACTION, PSTART_ACTION, \
REFRESH_STACK_HASH_ACTION, RESET_ACTION, RESTORE_ACTION, UPDATE_HOST_NAMES_ACTION, CHECK_DATABASE_ACTION, \
- SETUP_ACTION, SETUP_SECURITY_ACTION,START_ACTION, STATUS_ACTION, STOP_ACTION, RESTART_ACTION, UPGRADE_ACTION, \
- UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, SET_CURRENT_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, \
- UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, SET_CURRENT_ACTION, ENABLE_STACK_ACTION, SETUP_SSO_ACTION, \
+ SETUP_ACTION, SETUP_SECURITY_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, RESTART_ACTION, UPGRADE_ACTION, \
+ SETUP_JCE_ACTION, SET_CURRENT_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, \
+ SETUP_JCE_ACTION, SET_CURRENT_ACTION, ENABLE_STACK_ACTION, SETUP_SSO_ACTION, \
DB_PURGE_ACTION, INSTALL_MPACK_ACTION, UNINSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION, PAM_SETUP_ACTION, MIGRATE_LDAP_PAM_ACTION, KERBEROS_SETUP_ACTION
from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas, setup_pam, migrate_ldap_pam
from ambari_server.userInput import get_validated_string_input
@@ -753,7 +753,6 @@ def create_user_action_map(args, options):
RESET_ACTION: UserAction(reset, options),
STATUS_ACTION: UserAction(status, options),
UPGRADE_ACTION: UserAction(upgrade, options),
- UPGRADE_STACK_ACTION: UserActionPossibleArgs(upgrade_stack, [2, 4], args),
LDAP_SETUP_ACTION: UserAction(setup_ldap, options),
LDAP_SYNC_ACTION: UserAction(sync_ldap, options),
SET_CURRENT_ACTION: UserAction(set_current, options),
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd8d7109/ambari-server/src/main/python/ambari_server/setupActions.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/setupActions.py b/ambari-server/src/main/python/ambari_server/setupActions.py
index 926db98..61d20af 100644
--- a/ambari-server/src/main/python/ambari_server/setupActions.py
+++ b/ambari-server/src/main/python/ambari_server/setupActions.py
@@ -26,7 +26,6 @@ STOP_ACTION = "stop"
RESTART_ACTION = "restart"
RESET_ACTION = "reset"
UPGRADE_ACTION = "upgrade"
-UPGRADE_STACK_ACTION = "upgradestack"
REFRESH_STACK_HASH_ACTION = "refresh-stack-hash"
STATUS_ACTION = "status"
SETUP_HTTPS_ACTION = "setup-https"
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd8d7109/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index fb0bb70..5db7913 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -113,8 +113,8 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur
from ambari_server.serverSetup import check_selinux, check_ambari_user, proceedJDBCProperties, SE_STATUS_DISABLED, SE_MODE_ENFORCING, configure_os_settings, \
download_and_install_jdk, prompt_db_properties, setup, \
AmbariUserChecks, AmbariUserChecksLinux, AmbariUserChecksWindows, JDKSetup, reset, setup_jce_policy, expand_jce_zip_file
- from ambari_server.serverUpgrade import upgrade, upgrade_local_repo, change_objects_owner, upgrade_stack, \
- run_stack_upgrade, run_metainfo_upgrade, run_schema_upgrade, move_user_custom_actions, find_and_copy_custom_services
+ from ambari_server.serverUpgrade import upgrade, change_objects_owner, \
+ run_schema_upgrade, move_user_custom_actions, find_and_copy_custom_services
from ambari_server.setupHttps import is_valid_https_port, setup_https, import_cert_and_key_action, get_fqdn, \
generate_random_string, get_cert_info, COMMON_NAME_ATTR, is_valid_cert_exp, NOT_AFTER_ATTR, NOT_BEFORE_ATTR, \
SSL_DATE_FORMAT, import_cert_and_key, is_valid_cert_host, setup_truststore, \
@@ -4913,58 +4913,6 @@ class TestAmbariServer(TestCase):
self.assertEqual(shutil_move_mock.call_count, 2)
pass
-
- @patch("os.path.isdir", new = MagicMock(return_value=True))
- @patch("os.access", new = MagicMock(return_value=True))
- @patch("ambari_server.serverConfiguration.get_conf_dir")
- @patch("ambari_server.serverUpgrade.run_os_command")
- @patch("ambari_server.serverUpgrade.get_java_exe_path")
- def test_run_stack_upgrade(self, java_exe_path_mock, run_os_command_mock,
- get_conf_dir_mock):
- java_exe_path_mock.return_value = "/usr/lib/java/bin/java"
- run_os_command_mock.return_value = (0, None, None)
- get_conf_dir_mock.return_value = '/etc/conf'
- stackIdMap = {'HDP' : '2.0', 'repo_url' : 'http://test.com'}
-
- run_stack_upgrade(None, 'HDP', '2.0', 'http://test.com', None)
-
- self.assertTrue(java_exe_path_mock.called)
- self.assertTrue(get_conf_dir_mock.called)
- self.assertTrue(run_os_command_mock.called)
- run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp \'/etc/conf:/usr/lib/ambari-server/*\' '
- 'org.apache.ambari.server.upgrade.StackUpgradeHelper '
- 'updateStackId ' + "'" + json.dumps(stackIdMap) + "'" +
- ' > ' + os.sep + 'var' + os.sep + 'log' + os.sep + 'ambari-server' + os.sep +
- 'ambari-server.out 2>&1')
- pass
-
- @patch("os.path.isdir", new = MagicMock(return_value=True))
- @patch("os.access", new = MagicMock(return_value=True))
- @patch.object(ServerClassPath, "get_full_ambari_classpath_escaped_for_shell",
- new = MagicMock(return_value = '/etc/conf' + os.pathsep + 'test' + os.pathsep + 'path12'))
- @patch("ambari_server.serverConfiguration.get_conf_dir")
- @patch("ambari_server.serverUpgrade.run_os_command")
- @patch("ambari_server.serverUpgrade.get_java_exe_path")
- def test_run_stack_upgrade_with_url_os(self, java_exe_path_mock, run_os_command_mock,
- get_conf_dir_mock):
- java_exe_path_mock.return_value = "/usr/lib/java/bin/java"
- run_os_command_mock.return_value = (0, None, None)
- get_conf_dir_mock.return_value = '/etc/conf'
- stackIdMap = {'HDP' : '2.0', 'repo_url': 'http://test.com', 'repo_url_os': 'centos5,centos6'}
-
- run_stack_upgrade(None, 'HDP', '2.0', 'http://test.com', 'centos5,centos6')
-
- self.assertTrue(java_exe_path_mock.called)
- self.assertTrue(get_conf_dir_mock.called)
- self.assertTrue(run_os_command_mock.called)
- run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp /etc/conf' + os.pathsep + 'test:path12 '
- 'org.apache.ambari.server.upgrade.StackUpgradeHelper '
- 'updateStackId ' + "'" + json.dumps(stackIdMap) + "'" +
- ' > ' + os.sep + 'var' + os.sep + 'log' + os.sep + 'ambari-server' + os.sep +
- 'ambari-server.out 2>&1')
- pass
-
-
@patch("os.path.isdir", new = MagicMock(return_value=True))
@patch("os.access", new = MagicMock(return_value=True))
@patch.object(ServerClassPath, "get_full_ambari_classpath_escaped_for_shell",
@@ -5011,32 +4959,6 @@ class TestAmbariServer(TestCase):
self.assertTrue(run_os_command_mock.called)
run_os_command_mock.assert_called_with(command, env=environ)
- @patch("os.path.isdir", new = MagicMock(return_value=True))
- @patch("os.access", new = MagicMock(return_value=True))
- @patch("ambari_server.serverConfiguration.get_conf_dir")
- @patch.object(ServerClassPath, "get_full_ambari_classpath_escaped_for_shell", new = MagicMock(return_value = 'test' + os.pathsep + 'path12'))
- @patch("ambari_server.serverUpgrade.run_os_command")
- @patch("ambari_server.serverUpgrade.get_java_exe_path")
- def test_run_metainfo_upgrade(self, java_exe_path_mock, run_os_command_mock,
- get_conf_dir_mock):
- java_exe_path_mock.return_value = "/usr/lib/java/bin/java"
- run_os_command_mock.return_value = (0, None, None)
- get_conf_dir_mock.return_value = '/etc/conf'
-
- json_map = {'a': 'http://newurl'}
- run_metainfo_upgrade(None, json_map)
-
- self.assertTrue(java_exe_path_mock.called)
- self.assertTrue(run_os_command_mock.called)
- run_os_command_mock.assert_called_with('/usr/lib/java/bin/java '
- '-cp test' + os.pathsep + 'path12 '
- 'org.apache.ambari.server.upgrade.StackUpgradeHelper updateMetaInfo ' +
- "'" + json.dumps(json_map) + "'" +
- ' > ' + os.sep + 'var' + os.sep + 'log' + os.sep + 'ambari-server' +
- os.sep + 'ambari-server.out 2>&1')
- pass
-
-
@not_for_platform(PLATFORM_WINDOWS)
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
@patch("os.path.isfile")
@@ -5257,14 +5179,15 @@ class TestAmbariServer(TestCase):
@patch("ambari_server.dbConfiguration.get_ambari_properties")
@patch("ambari_server.serverConfiguration.get_ambari_properties")
@patch("ambari_server.serverUpgrade.get_ambari_properties")
- @patch("ambari_server.serverUpgrade.upgrade_local_repo")
@patch("ambari_server.serverUpgrade.move_user_custom_actions")
@patch("ambari_server.serverUpgrade.update_krb_jaas_login_properties")
@patch("ambari_server.serverUpgrade.update_ambari_env")
@patch("ambari_server.setupMpacks.get_replay_log_file")
@patch("ambari_server.serverUpgrade.logger")
@patch.object(PGConfig, "_change_db_files_owner", return_value=0)
- def test_upgrade_from_161(self, change_db_files_owner_mock, logger_mock, get_replay_log_file_mock, update_ambari_env_mock, update_krb_jaas_login_properties_mock, move_user_custom_actions_mock, upgrade_local_repo_mock, get_ambari_properties_mock,
+ def test_upgrade_from_161(self, change_db_files_owner_mock, logger_mock, get_replay_log_file_mock,
+ update_ambari_env_mock, update_krb_jaas_login_properties_mock,
+ move_user_custom_actions_mock, get_ambari_properties_mock,
get_ambari_properties_2_mock, get_ambari_properties_3_mock, get_ambari_version_mock, write_property_mock,
is_root_mock, update_ambari_properties_mock, find_properties_file_mock, run_os_command_mock,
run_schema_upgrade_mock, read_ambari_user_mock, print_warning_msg_mock,
@@ -5479,11 +5402,10 @@ class TestAmbariServer(TestCase):
@patch("ambari_server.dbConfiguration.get_ambari_properties")
@patch("ambari_server.serverConfiguration.get_ambari_properties")
@patch("ambari_server.serverUpgrade.get_ambari_properties")
- @patch("ambari_server.serverUpgrade.upgrade_local_repo")
@patch("ambari_server.serverUpgrade.move_user_custom_actions")
@patch("ambari_server.serverUpgrade.update_krb_jaas_login_properties")
@patch("ambari_server.serverUpgrade.update_ambari_env")
- def test_upgrade(self, update_ambari_env_mock, update_krb_jaas_login_properties_mock, move_user_custom_actions, upgrade_local_repo_mock,
+ def test_upgrade(self, update_ambari_env_mock, update_krb_jaas_login_properties_mock, move_user_custom_actions,
get_ambari_properties_mock, get_ambari_properties_2_mock, get_ambari_properties_3_mock,
is_root_mock, get_ambari_version_mock, get_ambari_version_2_mock,
parse_properties_file_mock,
@@ -5549,7 +5471,6 @@ class TestAmbariServer(TestCase):
self.assertTrue(print_warning_msg_mock.called)
warning_args = print_warning_msg_mock.call_args[0][0]
self.assertTrue("custom ambari user" in warning_args)
- self.assertTrue(upgrade_local_repo_mock.called)
self.assertTrue(move_user_custom_actions.called)
args = reset_mocks()
@@ -5664,10 +5585,9 @@ class TestAmbariServer(TestCase):
@patch("ambari_server.dbConfiguration.get_ambari_properties")
@patch("ambari_server.serverConfiguration.get_ambari_properties")
@patch("ambari_server.serverUpgrade.get_ambari_properties")
- @patch("ambari_server.serverUpgrade.upgrade_local_repo")
@patch("ambari_server.serverUpgrade.move_user_custom_actions")
@patch("ambari_server.serverUpgrade.update_krb_jaas_login_properties")
- def test_upgrade(self, update_krb_jaas_login_properties_mock, move_user_custom_actions, upgrade_local_repo_mock,
+ def test_upgrade(self, update_krb_jaas_login_properties_mock, move_user_custom_actions,
get_ambari_properties_mock, get_ambari_properties_2_mock, get_ambari_properties_3_mock,
is_root_mock, get_ambari_version_mock, get_ambari_version_2_mock,
parse_properties_file_mock,
@@ -5734,7 +5654,6 @@ class TestAmbariServer(TestCase):
self.assertTrue(print_warning_msg_mock.called)
warning_args = print_warning_msg_mock.call_args[0][0]
self.assertTrue("custom ambari user" in warning_args)
- self.assertTrue(upgrade_local_repo_mock.called)
self.assertTrue(move_user_custom_actions.called)
args = reset_mocks()
@@ -8313,79 +8232,6 @@ class TestAmbariServer(TestCase):
validatorFunction = is_valid_filepath, answer='/kerberos/admin.keytab')
pass
- @patch("os.listdir")
- @patch("os.path.exists")
- @patch("ambari_server.serverUpgrade.load_stack_values")
- @patch("ambari_server.serverUpgrade.get_ambari_properties")
- @patch("ambari_server.serverUpgrade.run_metainfo_upgrade")
- def test_upgrade_local_repo(self,
- run_metainfo_upgrade_mock,
- get_ambari_properties_mock,
- load_stack_values_mock,
- os_path_exists_mock,
- os_listdir_mock):
-
- from mock.mock import call
- args = MagicMock()
- args.persistence_type = "local"
-
- def load_values_side_effect(*args, **kwargs):
- res = {}
- res['a'] = 'http://oldurl'
- if -1 != args[1].find("HDPLocal"):
- res['a'] = 'http://newurl'
- return res
-
- load_stack_values_mock.side_effect = load_values_side_effect
-
- properties = Properties()
- get_ambari_properties_mock.return_value = properties
- os_path_exists_mock.return_value = 1
- os_listdir_mock.return_value = ['1.1']
-
- upgrade_local_repo(args)
-
- self.assertTrue(get_ambari_properties_mock.called)
- self.assertTrue(load_stack_values_mock.called)
- self.assertTrue(run_metainfo_upgrade_mock.called)
- run_metainfo_upgrade_mock.assert_called_with(args, {'a': 'http://newurl'})
- pass
-
- @patch("os.listdir")
- @patch("os.path.exists")
- @patch("ambari_server.serverUpgrade.load_stack_values")
- @patch("ambari_server.serverUpgrade.get_ambari_properties")
- @patch("ambari_server.serverUpgrade.run_metainfo_upgrade")
- def test_upgrade_local_repo_nochange(self,
- run_metainfo_upgrade_mock,
- get_ambari_properties_mock,
- load_stack_values_mock,
- os_path_exists_mock,
- os_listdir_mock):
-
- from mock.mock import call
- args = MagicMock()
- args.persistence_type = "local"
-
- def load_values_side_effect(*args, **kwargs):
- res = {}
- res['a'] = 'http://oldurl'
- return res
-
- load_stack_values_mock.side_effect = load_values_side_effect
-
- properties = Properties()
- get_ambari_properties_mock.return_value = properties
- os_path_exists_mock.return_value = 1
- os_listdir_mock.return_value = ['1.1']
-
- upgrade_local_repo(args)
-
- self.assertTrue(get_ambari_properties_mock.called)
- self.assertTrue(load_stack_values_mock.called)
- self.assertTrue(run_metainfo_upgrade_mock.called)
- run_metainfo_upgrade_mock.assert_called_with(args, {})
- pass
@patch("os.path.exists")
@patch.object(ResourceFilesKeeper, "perform_housekeeping")
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd8d7109/ambari-server/src/test/python/TestServerUpgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestServerUpgrade.py b/ambari-server/src/test/python/TestServerUpgrade.py
index 9235ff9..e61048a 100644
--- a/ambari-server/src/test/python/TestServerUpgrade.py
+++ b/ambari-server/src/test/python/TestServerUpgrade.py
@@ -36,7 +36,7 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur
with patch("os.path.isdir", return_value = MagicMock(return_value=True)):
with patch("os.access", return_value = MagicMock(return_value=True)):
with patch.object(os_utils, "parse_log4j_file", return_value={'ambari.log.dir': '/var/log/ambari-server'}):
- from ambari_server.serverUpgrade import set_current, SetCurrentVersionOptions, upgrade_stack
+ from ambari_server.serverUpgrade import set_current, SetCurrentVersionOptions
import ambari_server
os_utils.search_file = _search_file
@@ -154,40 +154,6 @@ class TestServerUpgrade(TestCase):
self.assertEquals(request.origin_req_host, '127.0.0.1')
self.assertEquals(request.headers, {'X-requested-by': 'ambari', 'Authorization': 'Basic ZHVtbXlfc3RyaW5nOmR1bW15X3N0cmluZw=='})
- @patch("ambari_server.serverUpgrade.run_os_command")
- @patch("ambari_server.serverUpgrade.get_java_exe_path")
- @patch("ambari_server.serverConfiguration.get_ambari_properties")
- @patch("ambari_server.serverUpgrade.get_ambari_properties")
- @patch("ambari_server.serverUpgrade.check_database_name_property")
- @patch("ambari_server.serverUpgrade.is_root")
- def test_upgrade_stack(self, is_root_mock, c_d_n_p_mock, up_g_a_p_mock, server_g_a_p_mock, java_path_mock, run_os_mock):
-
- run_os_mock.return_value = 0, "", ""
-
- java_path_mock.return_value = ""
-
- is_root_mock.return_value = True
-
- def do_nothing():
- pass
- c_d_n_p_mock.side_effect = do_nothing
-
- p = ambari_server.properties.Properties()
- p._props = {
- ambari_server.serverConfiguration.JDBC_DATABASE_PROPERTY: "mysql",
- ambari_server.serverConfiguration.JDBC_DATABASE_NAME_PROPERTY: "ambari"
- }
-
- up_g_a_p_mock.side_effect = [p, p]
- server_g_a_p_mock.side_effect = [p]
-
- args = ["upgrade_stack", "HDP-2.3"]
- upgrade_stack(args)
-
- self.assertTrue(run_os_mock.called)
- command = run_os_mock.call_args_list[0][0][0]
- self.assertTrue("StackUpgradeHelper" in command and "HDP" in command and "2.3" in command)
-
def testCurrentVersionOptions(self):
# Negative test cases
options = MagicMock()