You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2013/06/13 02:00:45 UTC
svn commit: r1492504 - in /incubator/ambari/trunk/ambari-server/src:
main/python/ambari-server.py test/python/TestAmbaryServer.py
Author: mahadev
Date: Thu Jun 13 00:00:45 2013
New Revision: 1492504
URL: http://svn.apache.org/r1492504
Log:
AMBARI-2349. Enhance processing of ojdbc.jar before starting ambari server
(Oleksandr via mahadev)
Modified:
incubator/ambari/trunk/ambari-server/src/main/python/ambari-server.py
incubator/ambari/trunk/ambari-server/src/test/python/TestAmbaryServer.py
Modified: incubator/ambari/trunk/ambari-server/src/main/python/ambari-server.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/python/ambari-server.py?rev=1492504&r1=1492503&r2=1492504&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/python/ambari-server.py (original)
+++ incubator/ambari/trunk/ambari-server/src/main/python/ambari-server.py Thu Jun 13 00:00:45 2013
@@ -78,6 +78,9 @@ JAVA_SHARE_PATH="/usr/share/java"
BOLD_ON='\033[1m'
BOLD_OFF='\033[0m'
+#Common messages
+PRESS_ENTER_MSG="Press <enter> to continue."
+
if ambari_provider_module is not None:
ambari_provider_module_option = "-Dprovider.module.class=" +\
ambari_provider_module + " "
@@ -160,6 +163,11 @@ DATABASE_PORTS =["5432", "1521", "3306"]
DATABASE_DRIVER_NAMES = ["org.postgresql.Driver", "oracle.jdbc.driver.OracleDriver", "com.mysql.jdbc.Driver"]
DATABASE_CONNECTION_STRINGS = ["jdbc:postgresql://{0}:{1}/{2}", "jdbc:oracle:thin:@{0}:{1}/{2}", "jdbc:mysql://{0}:{1}/{2}"]
DATABASE_CLI_TOOLS = [["psql"], ["sqlplus", "sqlplus64"], ["mysql"]]
+DATABASE_CLI_TOOLS_DESC = ["psql", "sqlplus", "mysql"]
+DATABASE_CLI_TOOLS_USAGE = ['su -postgres --command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'"',
+ 'sqlplus {1}/{2} < {0} ',
+ 'mysql --user={1} --password={2} {3}<{0}']
+
DATABASE_INIT_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-REMOTE-CREATE.sql',
'/var/lib/ambari-server/resources/Ambari-DDL-Oracle-CREATE.sql',
'/var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql']
@@ -551,13 +559,6 @@ def track_jdk(base_name, url, local_name
# Downloads the JDK
#
def download_jdk(args):
- conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
- if conf_file is None:
- print 'File %s not found in search path $%s: %s' %\
- (AMBARI_PROPERTIES_FILE, AMBARI_CONF_VAR, get_conf_dir())
- return -1
- print_info_msg('Loading properties from ' + conf_file)
-
if get_JAVA_HOME():
return 0
@@ -571,14 +572,11 @@ def download_jdk(args):
write_property(JAVA_HOME_PROPERTY, args.java_home)
return 0
- properties = None
- try:
- properties = Properties()
- properties.load(open(conf_file))
- except (Exception), e:
- print 'Could not read "%s": %s' % (conf_file, e)
+ properties = get_ambari_properties()
+ if properties == -1:
+ print_error_msg ("Error getting ambari properties")
return -1
-
+ conf_file = properties.fileName
try:
jdk_url = properties['jdk.url']
resources_dir = properties['resources.dir']
@@ -828,19 +826,11 @@ def update_ambari_properties():
# Configures the OS settings in ambari properties.
#
def configure_os_settings():
- conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
- if conf_file is None:
- print_error_msg ('File %s not found in search path $%s: %s'
- % (AMBARI_PROPERTIES_FILE, AMBARI_CONF_VAR, get_conf_dir()))
- return -1
- print_info_msg ('Loading properties from ' + conf_file)
- properties = None
- try:
- properties = Properties()
- properties.load(open(conf_file))
- except (Exception), e:
- print_error_msg ('Could not read "%s": %s' % (conf_file, e))
+ properties = get_ambari_properties()
+ if properties == -1:
+ print_error_msg ("Error getting ambari properties")
return -1
+
try:
conf_os_type = properties[OS_TYPE_PROPERTY]
if conf_os_type != '':
@@ -861,30 +851,23 @@ def configure_os_settings():
if os_name == 'suse':
os_name = 'sles'
os_version = os_info[1].split('.', 1)[0]
- master_os_type = os_name + os_version
- with open(conf_file, "a") as ambariConf:
- ambariConf.write(OS_TYPE_PROPERTY + "=" + master_os_type)
- ambariConf.write("\n")
- ambariConf.closed
+ master_os_type = os_name + os_version
+ write_property(OS_TYPE_PROPERTY, master_os_type)
return 0
def get_JAVA_HOME():
- conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
- properties = Properties()
-
- try:
- properties.load(open(conf_file))
- java_home = properties[JAVA_HOME_PROPERTY]
- if (not 0 == len(java_home)) and (os.path.exists(java_home)):
- return java_home
- except (Exception), e:
- print 'Could not read "%s": %s' % (conf_file, e)
-
- return None
-
+ properties = get_ambari_properties()
+ if properties == -1:
+ print_error_msg ("Error getting ambari properties")
+ return None
+
+ java_home = properties[JAVA_HOME_PROPERTY]
+ if (not 0 == len(java_home)) and (os.path.exists(java_home)):
+ return java_home
+ return None
#
# Finds the available JDKs.
@@ -982,9 +965,6 @@ def setup(args):
print_error_msg ('Failed to stop iptables. Exiting.')
sys.exit(retcode)
- print 'Configuring database...'
- prompt_db_properties(args)
-
print 'Checking JDK...'
retcode = download_jdk(args)
if not retcode == 0:
@@ -997,6 +977,9 @@ def setup(args):
print_error_msg ('Configure of OS settings in '
'ambari.properties failed. Exiting.')
sys.exit(retcode)
+
+ print 'Configuring database...'
+ prompt_db_properties(args)
#DB setup should be done last after doing any setup.
@@ -1080,30 +1063,36 @@ def reset(args):
print "Resetting the Server database..."
parse_properties_file(args)
-
# configure_database_username_password(args)
if args.persistence_type=="remote":
+ client_desc = DATABASE_NAMES[DATABASE_INDEX] + ' ' + DATABASE_CLI_TOOLS_DESC[DATABASE_INDEX]
+ client_usage_cmd_drop = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_DROP_SCRIPTS[DATABASE_INDEX], args.database_username,
+ args.database_password, args.database_name)
+ client_usage_cmd_init = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_INIT_SCRIPTS[DATABASE_INDEX], args.database_username,
+ args.database_password, args.database_name)
if get_db_cli_tool(args) != -1:
retcode, out, err = execute_remote_script(args, DATABASE_DROP_SCRIPTS[DATABASE_INDEX])
if not retcode == 0:
if retcode == -1:
- print_warning_msg('Cannot find ' + DATABASE_NAMES[DATABASE_INDEX] + ' client in the path, for reset please run the following DDL against the DB: ' +
- DATABASE_DROP_SCRIPTS[DATABASE_INDEX] + ", then: " + DATABASE_INIT_SCRIPTS[DATABASE_INDEX], True)
+ print_warning_msg('Cannot find ' + client_desc + ' client in the path to reset the Ambari Server schema. To reset Ambari Server schema ' +
+ 'you must run the following DDL against the database to drop the schema:' + os.linesep + client_usage_cmd_drop + os.linesep +
+ ', then you must run the following DDL against the database to create the schema ' + os.linesep + client_usage_cmd_init + os.linesep )
print err
return retcode
retcode, out, err = execute_remote_script(args, DATABASE_INIT_SCRIPTS[DATABASE_INDEX])
if not retcode == 0:
if retcode == -1:
- print_warning_msg('Cannot find ' + DATABASE_NAMES[DATABASE_INDEX] + ' client in the path, for reset please run the following DDL against the DB: ' +
- DATABASE_DROP_SCRIPTS[DATABASE_INDEX] + ", then: " + DATABASE_INIT_SCRIPTS[DATABASE_INDEX], True)
+ print_warning_msg('Cannot find ' + client_desc + ' client in the path to reset the Ambari Server schema. To reset Ambari Server schema ' +
+ 'you must run the following DDL against the database to drop the schema:' + os.linesep + client_usage_cmd_drop + os.linesep +
+ ', then you must run the following DDL against the database to create the schema ' + os.linesep + client_usage_cmd_init + os.linesep )
print err
return retcode
else:
- print_warning_msg('Cannot find ' + DATABASE_NAMES[DATABASE_INDEX] + ' client in the path, for reset please run the following DDL against the DB: ' +
- DATABASE_INIT_SCRIPTS[DATABASE_INDEX] + ", then: " + DATABASE_DROP_SCRIPTS[DATABASE_INDEX], True)
- print_error_msg(DATABASE_CLI_TOOLS[DATABASE_INDEX] + " not found. Unable to perform automatic reset.")
+ print_warning_msg('Cannot find ' + client_desc + ' client in the path to reset the Ambari Server schema. To reset Ambari Server schema ' +
+ 'you must run the following DDL against the database to drop the schema:' + os.linesep + client_usage_cmd_drop + os.linesep +
+ ', then you must run the following DDL against the database to create the schema ' + os.linesep + client_usage_cmd_init + os.linesep )
return -1
else:
@@ -1446,13 +1435,9 @@ def prompt_db_properties(args):
# Store set of properties for remote database connection
def store_remote_properties(args):
- conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
- properties = Properties()
-
- try:
- properties.load(open(conf_file))
- except Exception, e:
- print 'Could not read ambari config file "%s": %s' % (conf_file, e)
+ properties = get_ambari_properties()
+ if properties == -1:
+ print_error_msg ("Error getting ambari properties")
return -1
properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "remote")
@@ -1478,6 +1463,9 @@ def store_remote_properties(args):
properties.process_pair(JDBC_RCA_USER_NAME_PROPERTY, args.database_username)
properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY, store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
+
+ conf_file = properties.fileName
+
try:
properties.store(open(conf_file, "w"))
except Exception, e:
@@ -1488,14 +1476,21 @@ def store_remote_properties(args):
# Initialize remote database schema
def setup_remote_db(args):
+
+ not_found_msg = "Cannot find {0} {1} client in the path to load the Ambari Server schema.\
+ Before starting Ambari Server, you must run the following DDL against the database to create \
+the schema ".format(DATABASE_NAMES[DATABASE_INDEX], str(DATABASE_CLI_TOOLS_DESC[DATABASE_INDEX]))
+ client_usage_cmd = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_INIT_SCRIPTS[DATABASE_INDEX], args.database_username,
+ args.database_password, args.database_name)
+
retcode, out, err = execute_remote_script(args, DATABASE_INIT_SCRIPTS[DATABASE_INDEX])
if retcode != 0:
if retcode == -1:
- print_warning_msg('Cannot find ' + DATABASE_NAMES[DATABASE_INDEX] +
- ' client in the path, for setup please run the following DDL against the DB: ' +
- DATABASE_INIT_SCRIPTS[DATABASE_INDEX], True)
+ print_warning_msg(not_found_msg + os.linesep + client_usage_cmd)
+ if not SILENT:
+ raw_input(PRESS_ENTER_MSG)
return retcode
print err
@@ -1540,7 +1535,8 @@ def execute_remote_script(args, scriptPa
tool = get_db_cli_tool(args)
if not tool:
args.warnings.append('{0} not found. Please, run DDL script manually'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
- print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
+ if VERBOSE:
+ print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
return -1, "Client wasn't found", "Client wasn't found"
if args.database == "postgres":
@@ -1578,13 +1574,9 @@ def execute_remote_script(args, scriptPa
return -2, "Wrong database", "Wrong database"
def configure_database_username_password(args):
- conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
- properties = Properties()
-
- try:
- properties.load(open(conf_file))
- except Exception, e:
- print 'Could not read ambari config file "%s": %s' % (conf_file, e)
+ properties = get_ambari_properties()
+ if properties == -1:
+ print_error_msg ("Error getting ambari properties")
return -1
username = properties[JDBC_USER_NAME_PROPERTY]
@@ -1601,14 +1593,11 @@ def configure_database_username_password
# Store local database connection properties
def store_local_properties(args):
- conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
- properties = Properties()
-
- try:
- properties.load(open(conf_file))
- except Exception, e:
- print 'Could not read ambari config file "%s": %s' % (conf_file, e)
+ properties = get_ambari_properties()
+ if properties == -1:
+ print_error_msg ("Error getting ambari properties")
return -1
+
properties.removeOldProp(JDBC_SCHEMA_PROPERTY)
properties.removeOldProp(JDBC_HOSTNAME_PROPERTY)
properties.removeOldProp(JDBC_DATABASE_PROPERTY)
@@ -1622,6 +1611,8 @@ def store_local_properties(args):
properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "local")
properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username)
properties.process_pair(JDBC_PASSWORD_FILE_PROPERTY, store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
+
+ conf_file = properties.fileName
try:
properties.store(open(conf_file, "w"))
@@ -1921,6 +1912,7 @@ class Properties(object):
if stream.mode != 'r':
raise ValueError, 'Stream should be opened in read-only mode!'
try:
+ self.fileName = os.path.abspath(stream.name)
lines = stream.readlines()
self.__parse(lines)
except IOError, e:
Modified: incubator/ambari/trunk/ambari-server/src/test/python/TestAmbaryServer.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/test/python/TestAmbaryServer.py?rev=1492504&r1=1492503&r2=1492504&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/test/python/TestAmbaryServer.py (original)
+++ incubator/ambari/trunk/ambari-server/src/test/python/TestAmbaryServer.py Thu Jun 13 00:00:45 2013
@@ -630,21 +630,17 @@ class TestAmbariServer(TestCase):
@patch("os.stat")
@patch("os.path.isfile")
@patch("os.path.exists")
- @patch("__builtin__.open")
@patch.object(ambari_server, "track_jdk")
@patch.object(ambari_server, "get_YN_input")
@patch.object(ambari_server, "run_os_command")
- @patch.object(ambari_server, "Properties")
@patch.object(ambari_server, "write_property")
@patch.object(ambari_server, "print_info_msg")
@patch.object(ambari_server, "get_JAVA_HOME")
- @patch.object(ambari_server, "get_conf_dir")
- @patch.object(ambari_server, "search_file")
- def test_download_jdk(self, search_file_mock, get_conf_dir_mock,
+ @patch.object(ambari_server, "get_ambari_properties")
+ def test_download_jdk(self, get_ambari_properties_mock,
get_JAVA_HOME_mock, print_info_msg_mock,
- write_property_mock, Properties_mock,
- run_os_command_mock, get_YN_input_mock, track_jdk_mock,
- openMock, path_existsMock,
+ write_property_mock, run_os_command_mock,
+ get_YN_input_mock, track_jdk_mock, path_existsMock,
path_isfileMock, statMock):
out = StringIO.StringIO()
@@ -652,15 +648,16 @@ class TestAmbariServer(TestCase):
args = MagicMock()
args.java_home = "somewhere"
- search_file_mock.return_value = None
+ path_existsMock.return_value = False
+ get_JAVA_HOME_mock.return_value = False
+ get_ambari_properties_mock.return_value = -1
rcode = ambari_server.download_jdk(args)
self.assertEqual(-1, rcode)
- self.assertTrue(search_file_mock.called)
- self.assertTrue(get_conf_dir_mock.called)
+ self.assertTrue(get_ambari_properties_mock.called)
- search_file_mock.return_value = "something"
+ #search_file_mock.return_value = "something"
get_JAVA_HOME_mock.return_value = True
path_existsMock.return_value = True
rcode = ambari_server.download_jdk(args)
@@ -671,14 +668,9 @@ class TestAmbariServer(TestCase):
self.assertEqual(0, rcode)
self.assertTrue(write_property_mock.called)
- p = MagicMock()
- Properties_mock.return_value = p
- openMock.side_effect = Exception("test exception")
path_existsMock.return_value = False
- rcode = ambari_server.download_jdk(args)
- self.assertEqual(-1, rcode)
-
- openMock.side_effect = None
+ p = MagicMock()
+ get_ambari_properties_mock.return_value = p
p.__getitem__.side_effect = KeyError("test exception")
rcode = ambari_server.download_jdk(args)
self.assertEqual(-1, rcode)
@@ -740,30 +732,21 @@ class TestAmbariServer(TestCase):
@patch("platform.linux_distribution")
@patch("platform.system")
- @patch("__builtin__.open")
- @patch.object(ambari_server, "Properties")
@patch.object(ambari_server, "print_info_msg")
@patch.object(ambari_server, "print_error_msg")
- @patch.object(ambari_server, "search_file")
+ @patch.object(ambari_server, "get_ambari_properties")
+ @patch.object(ambari_server, "write_property")
@patch.object(ambari_server, "get_conf_dir")
- def test_configure_os_settings(self, get_conf_dir_mock, search_file_mock,
- print_error_msg_mock, print_info_msg_mock,
- Properties_mock, openMock, systemMock,
- distMock):
+ def test_configure_os_settings(self, get_conf_dir_mock, write_property_mock, get_ambari_properties_mock, print_error_msg_mock,
+ print_info_msg_mock, systemMock, distMock):
- search_file_mock.return_value = None
+ get_ambari_properties_mock.return_value = -1
rcode = ambari_server.configure_os_settings()
self.assertEqual(-1, rcode)
-
- search_file_mock.return_value = "something"
+
p = MagicMock()
- Properties_mock.return_value = p
- openMock.side_effect = Exception("exception")
- rcode = ambari_server.configure_os_settings()
- self.assertEqual(-1, rcode)
-
- p.__getitem__.return_value = "something"
- openMock.side_effect = None
+ p[ambari_server.OS_TYPE_PROPERTY] = 'somevalue'
+ get_ambari_properties_mock.return_value = p
rcode = ambari_server.configure_os_settings()
self.assertEqual(0, rcode)
@@ -774,10 +757,9 @@ class TestAmbariServer(TestCase):
systemMock.return_value = "Linux"
distMock.return_value = ("CentOS", "6.3", None)
- f = MagicMock()
- openMock.return_value = f
rcode = ambari_server.configure_os_settings()
self.assertEqual(0, rcode)
+ self.assertTrue(write_property_mock.called)
@@ -1102,13 +1084,14 @@ class TestAmbariServer(TestCase):
@patch("sys.exit")
+ @patch.object(ambari_server, "download_jdk")
@patch.object(ambari_server, "get_db_cli_tool")
@patch.object(ambari_server, "store_remote_properties")
@patch.object(ambari_server, "is_local_database")
@patch.object(ambari_server, "check_iptables")
@patch.object(ambari_server, "check_jdbc_drivers")
def test_setup_remote_db_wo_client(self, check_jdbc_drivers_mock, check_iptables_mock, is_local_db_mock,
- store_remote_properties_mock, get_db_cli_tool_mock, exit_mock):
+ store_remote_properties_mock, get_db_cli_tool_mock, download_jdk_mock, exit_mock):
out = StringIO.StringIO()
sys.stdout = out
@@ -1217,6 +1200,101 @@ class TestAmbariServer(TestCase):
sys.stdout = sys.__stdout__
+ @patch.object(ambari_server, "get_ambari_properties")
+ @patch.object(ambari_server, "find_jdbc_driver")
+ @patch.object(ambari_server, "copy_files")
+ @patch.object(ambari_server, "print_error_msg")
+ @patch.object(ambari_server, "print_warning_msg")
+ @patch('__builtin__.raw_input')
+ @patch("sys.exit")
+ def test_check_jdbc_drivers(self, exit_mock, raw_input_mock, print_warning_msg, print_error_msg_mock, copy_files_mock,
+ find_jdbc_driver_mock, get_ambari_properties_mock):
+
+ out = StringIO.StringIO()
+ sys.stdout = out
+
+ args = MagicMock()
+
+ # Check positive scenario
+ drivers_list = ['driver_file']
+ resources_dir = '/tmp'
+
+ get_ambari_properties_mock.return_value = {ambari_server.RESOURCES_DIR_KEY : resources_dir}
+ find_jdbc_driver_mock.return_value = drivers_list
+
+ args.database = "oracle"
+
+ rcode = ambari_server.check_jdbc_drivers(args)
+
+ self.assertEqual(0, rcode)
+ copy_files_mock.assert_called_with(drivers_list, resources_dir)
+
+ # Check negative scenarios
+ # Silent option, no drivers
+ ambari_server.SILENT = True
+
+ find_jdbc_driver_mock.return_value = -1
+
+ rcode = ambari_server.check_jdbc_drivers(args)
+
+ self.assertTrue(print_error_msg_mock.called)
+ self.assertTrue(exit_mock.called)
+
+ # Non-Silent option, no drivers
+ ambari_server.SILENT = False
+
+ find_jdbc_driver_mock.return_value = -1
+
+ rcode = ambari_server.check_jdbc_drivers(args)
+
+ self.assertTrue(exit_mock.called)
+ self.assertTrue(print_error_msg_mock.called)
+
+ # Non-Silent option, no drivers at first ask, present drivers after that
+
+ find_jdbc_driver_mock.side_effect = [-1, drivers_list]
+
+ rcode = ambari_server.check_jdbc_drivers(args)
+
+ self.assertEqual(0, rcode)
+ copy_files_mock.assert_called_with(drivers_list, resources_dir)
+
+ # Non-Silent option, no drivers at first ask, present drivers after that
+ find_jdbc_driver_mock.reset()
+ find_jdbc_driver_mock.side_effect = [-1, -1]
+
+ rcode = ambari_server.check_jdbc_drivers(args)
+
+ self.assertTrue(exit_mock.called)
+ self.assertTrue(print_error_msg_mock.called)
+
+
+ sys.stdout = sys.__stdout__
+
+
+ @patch.object(ambari_server, "search_file")
+ def test_get_ambari_properties(self, search_file_mock):
+
+ search_file_mock.return_value = None
+ rcode = ambari_server.get_ambari_properties()
+ self.assertEqual(rcode, -1)
+
+ tf1 = tempfile.NamedTemporaryFile()
+ search_file_mock.return_value = tf1.name
+ prop_name='name'
+ prop_value='val'
+
+ with open(tf1.name, 'w') as fout:
+ fout.write(prop_name + '=' + prop_value)
+ fout.close()
+
+ properties = ambari_server.get_ambari_properties()
+
+ self.assertEqual(properties[prop_name], prop_value)
+ self.assertEqual(properties.fileName, os.path.abspath(tf1.name))
+
+ sys.stdout = sys.__stdout__
+
@patch.object(ambari_server, "search_file")
def test_parse_properties_file(self, search_file_mock):