You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/06/22 19:17:38 UTC
ambari git commit: AMBARI-12072 Ambari Upgrade: 2.0.1 -> 2.1.0 fails
with encrypt pwds option (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk d8724c077 -> fc1422a7e
AMBARI-12072 Ambari Upgrade: 2.0.1 -> 2.1.0 fails with encrypt pwds option (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fc1422a7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fc1422a7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fc1422a7
Branch: refs/heads/trunk
Commit: fc1422a7e05e2a6f7d1d9b1baa0e6d4e700b8eea
Parents: d8724c0
Author: Dmytro Sen <ds...@apache.org>
Authored: Mon Jun 22 20:16:29 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Mon Jun 22 20:16:29 2015 +0300
----------------------------------------------------------------------
.../main/python/ambari_server/serverUpgrade.py | 10 ++-
.../main/python/ambari_server/setupSecurity.py | 75 ++++++++++++++++-
.../src/main/python/ambari_server_main.py | 85 +++-----------------
.../src/test/python/TestAmbariServer.py | 52 +++++++++---
4 files changed, 132 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/fc1422a7/ambari-server/src/main/python/ambari_server/serverUpgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverUpgrade.py b/ambari-server/src/main/python/ambari_server/serverUpgrade.py
index 12a82fa..6315f9a 100644
--- a/ambari-server/src/main/python/ambari_server/serverUpgrade.py
+++ b/ambari-server/src/main/python/ambari_server/serverUpgrade.py
@@ -36,7 +36,8 @@ from ambari_server.serverConfiguration import configDefaults, \
update_database_name_property, get_admin_views_dir, \
AMBARI_PROPERTIES_FILE, IS_LDAP_CONFIGURED, LDAP_PRIMARY_URL_PROPERTY, RESOURCES_DIR_PROPERTY, \
SETUP_OR_UPGRADE_MSG
-from ambari_server.setupSecurity import adjust_directory_permissions
+from ambari_server.setupSecurity import adjust_directory_permissions, \
+ generate_env, ensure_can_start_under_current_user
from ambari_server.utils import compare_versions
from ambari_server.serverUtils import is_server_runing, get_ambari_server_api_base
from ambari_server.userInput import get_validated_string_input, get_prompt_default, read_password, get_YN_input
@@ -223,7 +224,12 @@ def run_schema_upgrade():
print 'Upgrading database schema'
command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath())
- (retcode, stdout, stderr) = run_os_command(command)
+
+ ambari_user = read_ambari_user()
+ current_user = ensure_can_start_under_current_user(ambari_user)
+ environ = generate_env(ambari_user, current_user)
+
+ (retcode, stdout, stderr) = run_os_command(command, env=environ)
print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
if retcode > 0:
print_error_msg("Error executing schema upgrade, please check the server logs.")
http://git-wip-us.apache.org/repos/asf/ambari/blob/fc1422a7/ambari-server/src/main/python/ambari_server/setupSecurity.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/setupSecurity.py b/ambari-server/src/main/python/ambari_server/setupSecurity.py
index 1bc2222..6bb8e4b 100644
--- a/ambari-server/src/main/python/ambari_server/setupSecurity.py
+++ b/ambari-server/src/main/python/ambari_server/setupSecurity.py
@@ -19,6 +19,9 @@ limitations under the License.
'''
import base64
import fileinput
+import getpass
+import stat
+import tempfile
import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
import os
import re
@@ -45,7 +48,7 @@ from ambari_server.serverConfiguration import configDefaults, \
SECURITY_PROVIDER_KEY_CMD, SECURITY_MASTER_KEY_FILENAME, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
SSL_TRUSTSTORE_PASSWORD_PROPERTY, SSL_TRUSTSTORE_PATH_PROPERTY, SSL_TRUSTSTORE_TYPE_PROPERTY, \
SSL_API, SSL_API_PORT, DEFAULT_SSL_API_PORT, CLIENT_API_PORT, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY, JAVA_HOME_PROPERTY, \
- get_resources_location
+ get_resources_location, SECURITY_MASTER_KEY_LOCATION, SETUP_OR_UPGRADE_MSG
from ambari_server.serverUtils import is_server_runing, get_ambari_server_api_base
from ambari_server.setupActions import SETUP_ACTION, LDAP_SETUP_ACTION
from ambari_server.userInput import get_validated_string_input, get_prompt_default, read_password, get_YN_input
@@ -703,3 +706,73 @@ def setup_ldap():
print 'Saving...done'
return 0
+
+@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
+def generate_env(ambari_user, current_user):
+ properties = get_ambari_properties()
+ isSecure = get_is_secure(properties)
+ (isPersisted, masterKeyFile) = get_is_persisted(properties)
+ environ = os.environ.copy()
+ # Need to handle master key not persisted scenario
+ if isSecure and not masterKeyFile:
+ prompt = False
+ masterKey = environ.get(SECURITY_KEY_ENV_VAR_NAME)
+
+ if masterKey is not None and masterKey != "":
+ pass
+ else:
+ keyLocation = environ.get(SECURITY_MASTER_KEY_LOCATION)
+
+ if keyLocation is not None:
+ try:
+ # Verify master key can be read by the java process
+ with open(keyLocation, 'r'):
+ pass
+ except IOError:
+ print_warning_msg("Cannot read Master key from path specified in "
+ "environemnt.")
+ prompt = True
+ else:
+ # Key not provided in the environment
+ prompt = True
+
+ if prompt:
+ import pwd
+
+ masterKey = get_original_master_key(properties)
+ tempDir = tempfile.gettempdir()
+ tempFilePath = tempDir + os.sep + "masterkey"
+ save_master_key(masterKey, tempFilePath, True)
+ if ambari_user != current_user:
+ uid = pwd.getpwnam(ambari_user).pw_uid
+ gid = pwd.getpwnam(ambari_user).pw_gid
+ os.chown(tempFilePath, uid, gid)
+ else:
+ os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
+
+ if tempFilePath is not None:
+ environ[SECURITY_MASTER_KEY_LOCATION] = tempFilePath
+
+ return environ
+
+@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
+def generate_env(ambari_user, current_user):
+ return os.environ.copy()
+
+@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
+def ensure_can_start_under_current_user(ambari_user):
+ #Ignore the requirement to run as root. In Windows, by default the child process inherits the security context
+ # and the environment from the parent process.
+ return ""
+
+@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
+def ensure_can_start_under_current_user(ambari_user):
+ current_user = getpass.getuser()
+ if ambari_user is None:
+ err = "Unable to detect a system user for Ambari Server.\n" + SETUP_OR_UPGRADE_MSG
+ raise FatalException(1, err)
+ if current_user != ambari_user and not is_root():
+ err = "Unable to start Ambari Server as user {0}. Please either run \"ambari-server start\" " \
+ "command as root, as sudo or as user \"{1}\"".format(current_user, ambari_user)
+ raise FatalException(1, err)
+ return current_user
http://git-wip-us.apache.org/repos/asf/ambari/blob/fc1422a7/ambari-server/src/main/python/ambari_server_main.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server_main.py b/ambari-server/src/main/python/ambari_server_main.py
index 35201cd..bc561a1 100644
--- a/ambari-server/src/main/python/ambari_server_main.py
+++ b/ambari-server/src/main/python/ambari_server_main.py
@@ -17,11 +17,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
-import getpass
import os
-import stat
import subprocess
-import tempfile
import sys
from ambari_commons.exceptions import FatalException
@@ -37,7 +34,8 @@ from ambari_server.serverConfiguration import configDefaults, find_jdk, get_amba
SETUP_OR_UPGRADE_MSG, check_database_name_property, parse_properties_file
from ambari_server.serverUtils import refresh_stack_hash
from ambari_server.setupHttps import get_fqdn
-from ambari_server.setupSecurity import save_master_key
+from ambari_server.setupSecurity import generate_env, \
+ ensure_can_start_under_current_user
from ambari_server.utils import check_reverse_lookup, save_pid, locate_file, looking_for_pid, wait_for_pid, \
save_main_pid_ex, check_exitcode
@@ -107,26 +105,6 @@ AMBARI_SERVER_DIE_MSG = "Ambari Server java process died with exitcode {0}. Chec
ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files'
ULIMIT_OPEN_FILES_DEFAULT = 10000
-
-@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
-def ensure_can_start_under_current_user(ambari_user):
- #Ignore the requirement to run as root. In Windows, by default the child process inherits the security context
- # and the environment from the parent process.
- return ""
-
-@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
-def ensure_can_start_under_current_user(ambari_user):
- current_user = getpass.getuser()
- if ambari_user is None:
- err = "Unable to detect a system user for Ambari Server.\n" + SETUP_OR_UPGRADE_MSG
- raise FatalException(1, err)
- if current_user != ambari_user and not is_root():
- err = "Unable to start Ambari Server as user {0}. Please either run \"ambari-server start\" " \
- "command as root, as sudo or as user \"{1}\"".format(current_user, ambari_user)
- raise FatalException(1, err)
- return current_user
-
-
@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
def ensure_server_security_is_configured():
pass
@@ -144,7 +122,7 @@ def get_ulimit_open_files(properties):
return open_files
@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
-def generate_child_process_param_list(ambari_user, current_user, java_exe, class_path, debug_start, suspend_mode):
+def generate_child_process_param_list(ambari_user, java_exe, class_path, debug_start, suspend_mode):
conf_dir = class_path
if class_path.find(' ') != -1:
conf_dir = '"' + class_path + '"'
@@ -155,58 +133,15 @@ def generate_child_process_param_list(ambari_user, current_user, java_exe, class
jvm_args,
conf_dir,
suspend_mode)
- environ = os.environ.copy()
- return (command, environ)
+ return command
@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
-def generate_child_process_param_list(ambari_user, current_user, java_exe, class_path, debug_start, suspend_mode):
+def generate_child_process_param_list(ambari_user, java_exe, class_path,
+ debug_start, suspend_mode):
from ambari_commons.os_linux import ULIMIT_CMD
properties = get_ambari_properties()
- isSecure = get_is_secure(properties)
- (isPersisted, masterKeyFile) = get_is_persisted(properties)
- environ = os.environ.copy()
- # Need to handle master key not persisted scenario
- if isSecure and not masterKeyFile:
- prompt = False
- masterKey = environ.get(SECURITY_KEY_ENV_VAR_NAME)
-
- if masterKey is not None and masterKey != "":
- pass
- else:
- keyLocation = environ.get(SECURITY_MASTER_KEY_LOCATION)
-
- if keyLocation is not None:
- try:
- # Verify master key can be read by the java process
- with open(keyLocation, 'r'):
- pass
- except IOError:
- print_warning_msg("Cannot read Master key from path specified in "
- "environemnt.")
- prompt = True
- else:
- # Key not provided in the environment
- prompt = True
-
- if prompt:
- import pwd
-
- masterKey = get_original_master_key(properties)
- tempDir = tempfile.gettempdir()
- tempFilePath = tempDir + os.sep + "masterkey"
- save_master_key(masterKey, tempFilePath, True)
- if ambari_user != current_user:
- uid = pwd.getpwnam(ambari_user).pw_uid
- gid = pwd.getpwnam(ambari_user).pw_gid
- os.chown(tempFilePath, uid, gid)
- else:
- os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
-
- if tempFilePath is not None:
- environ[SECURITY_MASTER_KEY_LOCATION] = tempFilePath
-
command_base = SERVER_START_CMD_DEBUG if debug_start else SERVER_START_CMD
ulimit_cmd = "%s %s" % (ULIMIT_CMD, str(get_ulimit_open_files(properties)))
@@ -235,7 +170,7 @@ def generate_child_process_param_list(ambari_user, current_user, java_exe, class
cmd = "{ulimit_cmd} ; {command}".format(ulimit_cmd=ulimit_cmd, command=command)
param_list.append(cmd)
- return (param_list, environ)
+ return param_list
@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
def wait_for_server_start(pidFile, scmStatus):
@@ -329,8 +264,10 @@ def server_process_main(options, scmStatus=None):
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
- (param_list, environ) = generate_child_process_param_list(ambari_user, current_user,
- java_exe, class_path, debug_start, suspend_mode)
+ param_list = generate_child_process_param_list(ambari_user, java_exe,
+ class_path, debug_start,
+ suspend_mode)
+ environ = generate_env(ambari_user, current_user)
if not os.path.exists(configDefaults.PID_DIR):
os.makedirs(configDefaults.PID_DIR, 0755)
http://git-wip-us.apache.org/repos/asf/ambari/blob/fc1422a7/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 430fa60..777a63a 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -87,7 +87,7 @@ with patch("platform.linux_distribution", return_value = os_distro_value):
SRVR_ONE_WAY_SSL_PORT_PROPERTY, SRVR_TWO_WAY_SSL_PORT_PROPERTY, GANGLIA_HTTPS
from ambari_server.setupSecurity import adjust_directory_permissions, get_alias_string, get_ldap_event_spec_names, sync_ldap, LdapSyncOptions, \
configure_ldap_password, setup_ldap, REGEX_HOSTNAME_PORT, REGEX_TRUE_FALSE, REGEX_ANYTHING, setup_master_key, \
- setup_ambari_krb5_jaas
+ setup_ambari_krb5_jaas, ensure_can_start_under_current_user, generate_env
from ambari_server.userInput import get_YN_input, get_choice_string_input, get_validated_string_input, \
read_password
from ambari_server_main import get_ulimit_open_files, ULIMIT_OPEN_FILES_KEY, ULIMIT_OPEN_FILES_DEFAULT
@@ -3287,7 +3287,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
@patch.object(_ambari_server_, "is_server_runing")
@patch("os.chown")
@patch("ambari_server.setupSecurity.get_master_key_location")
- @patch("ambari_server_main.save_master_key")
+ @patch("ambari_server.setupSecurity.save_master_key")
@patch("ambari_server_main.get_is_persisted")
@patch("ambari_server_main.get_is_secure")
@patch('os.chmod', autospec=True)
@@ -3308,6 +3308,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
@patch("ambari_server_main.print_info_msg")
@patch.object(PGConfig, "_check_postgre_up")
@patch("ambari_server_main.read_ambari_user")
+ @patch("ambari_server.setupSecurity.is_root")
@patch("ambari_server.dbConfiguration_linux.is_root")
@patch("ambari_server_main.is_root")
@patch.object(LinuxDBMSConfig, "_find_jdbc_driver")
@@ -3315,7 +3316,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
@patch("os.chdir")
@patch.object(ResourceFilesKeeper, "perform_housekeeping")
def test_start(self, perform_housekeeping_mock, chdir_mock, getuser_mock, find_jdbc_driver_mock,
- is_root_mock, is_root_2_mock, read_ambari_user_mock,
+ is_root_mock, is_root_2_mock, is_root_3_mock, read_ambari_user_mock,
check_postgre_up_mock, print_info_msg_mock, print_warning_msg_mock,
find_jdk_mock, check_database_name_property_mock, search_file_mock,
popenMock, openMock, pexistsMock,
@@ -3408,7 +3409,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
args = reset_mocks()
read_ambari_user_mock.return_value = "dummy-user"
getuser_mock.return_value = "non_custom_user"
- is_root_2_mock.return_value = is_root_mock.return_value = False
+ is_root_3_mock.return_value = \
+ is_root_2_mock.return_value = \
+ is_root_mock.return_value = False
try:
_ambari_server_.start(args)
self.fail("Should fail with 'Can not start ambari-server as user...'")
@@ -3419,7 +3422,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
# Checking "jdk not found"
args = reset_mocks()
- is_root_2_mock.return_value = is_root_mock.return_value = True
+ is_root_3_mock.return_value = \
+ is_root_2_mock.return_value = \
+ is_root_mock.return_value = True
find_jdk_mock.return_value = None
try:
@@ -3433,7 +3438,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
find_jdk_mock.return_value = "somewhere"
## Testing workflow under root
- is_root_2_mock.return_value = is_root_mock.return_value = True
+ is_root_3_mock.return_value = \
+ is_root_2_mock.return_value = \
+ is_root_mock.return_value = True
# Remote DB
p.process_pair(JDBC_DATABASE_PROPERTY, 'oracle')
@@ -3571,7 +3578,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
popenMock.reset_mock()
## Testing workflow under non-root
- is_root_2_mock.return_value = is_root_mock.return_value = False
+ is_root_3_mock.return_value = \
+ is_root_2_mock.return_value = \
+ is_root_mock.return_value = False
read_ambari_user_mock.return_value = "not-root-user"
getuser_mock.return_value = read_ambari_user_mock.return_value
@@ -3626,7 +3635,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
p.process_pair(JDBC_DATABASE_PROPERTY, 'postgres')
p.process_pair(PERSISTENCE_TYPE_PROPERTY, 'local')
read_ambari_user_mock.return_value = "root"
- is_root_2_mock.return_value = is_root_mock.return_value = True
+ is_root_3_mock.return_value = \
+ is_root_2_mock.return_value = \
+ is_root_mock.return_value = True
_ambari_server_.start(args)
@@ -3645,7 +3656,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
p.process_pair(JDBC_DATABASE_PROPERTY, 'postgres')
p.process_pair(PERSISTENCE_TYPE_PROPERTY, 'local')
read_ambari_user_mock.return_value = "root"
- is_root_2_mock.return_value = is_root_mock.return_value = True
+ is_root_3_mock.return_value = \
+ is_root_2_mock.return_value = \
+ is_root_mock.return_value = True
get_validated_string_input_method.return_value = "masterkey"
os_chmod_method.return_value = None
get_is_secure_mock.return_value = True
@@ -3820,26 +3833,39 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
pass
+ @patch("ambari_server.serverUpgrade.ensure_can_start_under_current_user")
+ @patch("ambari_server.serverUpgrade.generate_env")
+ @patch("ambari_server.serverUpgrade.read_ambari_user")
@patch("ambari_server.serverConfiguration.get_conf_dir")
@patch("ambari_server.serverConfiguration.get_ambari_classpath")
@patch("ambari_server.serverUpgrade.run_os_command")
@patch("ambari_server.serverUpgrade.get_java_exe_path")
def test_run_schema_upgrade(self, java_exe_path_mock, run_os_command_mock,
- get_ambari_classpath_mock, get_conf_dir_mock):
+ get_ambari_classpath_mock, get_conf_dir_mock,
+ read_ambari_user_mock, generate_env_mock,
+ ensure_can_start_under_current_user_mock):
java_exe_path_mock.return_value = "/usr/lib/java/bin/java"
run_os_command_mock.return_value = (0, None, None)
get_ambari_classpath_mock.return_value = 'test:path12'
get_conf_dir_mock.return_value = '/etc/conf'
+ command = '/usr/lib/java/bin/java -cp /etc/conf:test:path12 ' \
+ 'org.apache.ambari.server.upgrade.SchemaUpgradeHelper ' \
+ '> /var/log/ambari-server/ambari-server.out 2>&1'
+ environ = {}
+ generate_env_mock.return_value = environ
+ ensure_can_start_under_current_user_mock.return_value = "root"
+ read_ambari_user_mock.return_value = "ambari"
run_schema_upgrade()
self.assertTrue(java_exe_path_mock.called)
+ self.assertTrue(ensure_can_start_under_current_user_mock.called)
+ self.assertTrue(generate_env_mock.called)
+ self.assertTrue(read_ambari_user_mock.called)
self.assertTrue(get_ambari_classpath_mock.called)
self.assertTrue(get_conf_dir_mock.called)
self.assertTrue(run_os_command_mock.called)
- run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp /etc/conf:test:path12 '
- 'org.apache.ambari.server.upgrade.SchemaUpgradeHelper '
- '> /var/log/ambari-server/ambari-server.out 2>&1')
+ run_os_command_mock.assert_called_with(command, env=environ)
pass