You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by fb...@apache.org on 2014/12/19 20:51:06 UTC
[1/7] ambari git commit: AMBARI-8373 Refactor the OS-dependent Ambari
Server Windows components
Repository: ambari
Updated Branches:
refs/heads/trunk bee937981 -> 2914d681d
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/test/python/custom_actions/TestCheckHost.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestCheckHost.py b/ambari-server/src/test/python/custom_actions/TestCheckHost.py
index cb934c9..567c032 100644
--- a/ambari-server/src/test/python/custom_actions/TestCheckHost.py
+++ b/ambari-server/src/test/python/custom_actions/TestCheckHost.py
@@ -17,19 +17,19 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
-
from stacks.utils.RMFTestCase import *
import json
import os
import socket
import subprocess
-from ambari_commons import inet_utils
-from resource_management import Script, ConfigDictionary
from mock.mock import patch
from mock.mock import MagicMock
-from unittest import TestCase
-from check_host import CheckHost
+with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
+ from resource_management import Script,ConfigDictionary
+ from unittest import TestCase
+
+ from check_host import CheckHost
class TestCheckHost(TestCase):
@@ -231,7 +231,7 @@ class TestCheckHost(TestCase):
self.assertTrue(structured_out_mock.called)
structured_out_mock.assert_called_with({})
-
+ @patch("platform.linux_distribution", return_value = ('Suse','11','Final'))
@patch.object(Script, 'get_config')
@patch.object(Script, 'get_tmp_dir')
@patch('resource_management.libraries.script.Script.put_structured_out')
@@ -244,7 +244,7 @@ class TestCheckHost(TestCase):
@patch('time.time')
def testLastAgentEnv(self, time_mock, checkReverseLookup_mock, checkIptables_mock, getTransparentHugePage_mock,
getUMask_mock, checkLiveServices_mock, javaProcs_mock, put_structured_out_mock,
- get_tmp_dir_mock, get_config_mock):
+ get_tmp_dir_mock, get_config_mock, platform_linux_distribution_mock):
jsonFilePath = os.path.join("../resources/custom_actions", "check_last_agent_env.json")
with open(jsonFilePath, "r") as jsonFile:
jsonPayload = json.load(jsonFile)
[5/7] ambari git commit: AMBARI-8373 Refactor the OS-dependent Ambari
Server Windows components
Posted by fb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py b/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py
index ce47fae..a4295f7 100644
--- a/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py
+++ b/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py
@@ -17,204 +17,177 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
-
+import fileinput
+import glob
+import os
+import re
import shutil
+import socket
+import subprocess
+import sys
+import time
+
+from ambari_commons import OSCheck, OSConst
+from ambari_commons.logging_utils import print_error_msg, print_info_msg, print_warning_msg, VERBOSE
+from ambari_commons.exceptions import NonFatalException, FatalException
+from ambari_commons.os_utils import copy_files, remove_file, run_os_command
+from ambari_server.dbConfiguration import DBMSConfig, USERNAME_PATTERN, SETUP_DB_CONNECT_ATTEMPTS, \
+ SETUP_DB_CONNECT_TIMEOUT
+from ambari_server.serverConfiguration import get_ambari_properties, get_value_from_properties, configDefaults, \
+ OS_TYPE, AMBARI_PROPERTIES_FILE, BLIND_PASSWORD, RESOURCES_DIR_PROPERTY, \
+ JDBC_DATABASE_PROPERTY, JDBC_DATABASE_NAME_PROPERTY, JDBC_POSTGRES_SCHEMA_PROPERTY, \
+ JDBC_HOSTNAME_PROPERTY, JDBC_PORT_PROPERTY, \
+ JDBC_USER_NAME_PROPERTY, JDBC_PASSWORD_PROPERTY, JDBC_PASSWORD_FILENAME, \
+ JDBC_DRIVER_PROPERTY, JDBC_URL_PROPERTY, \
+ JDBC_RCA_USER_NAME_PROPERTY, JDBC_RCA_PASSWORD_ALIAS, JDBC_RCA_PASSWORD_FILE_PROPERTY, \
+ JDBC_RCA_DRIVER_PROPERTY, JDBC_RCA_URL_PROPERTY, \
+ PERSISTENCE_TYPE_PROPERTY
+from ambari_server.setupSecurity import read_password, store_password_file, encrypt_password
+from ambari_server.userInput import get_YN_input, get_validated_string_input
+from ambari_server.utils import get_postgre_hba_dir, get_postgre_running_status
+
+_DEFAULT_PASSWORD = "bigdata"
+
+ORACLE_DB_ID_TYPES = ["Service Name", "SID"]
+
+JDBC_PROPERTIES_PREFIX = "server.jdbc.properties."
+
+class LinuxDBMSConfig(DBMSConfig):
+ def __init__(self, options, properties, storage_type):
+ super(LinuxDBMSConfig, self).__init__(options, properties, storage_type)
-from ambari_commons import OSConst
-from ambari_commons.logging_utils import *
-from exceptions import *
-from dbConfiguration import *
-from utils import *
-
-import utils
-
-# PostgreSQL settings
-PG_JDBC_CONNECTION_STRING = "jdbc:postgresql://{0}:{1}/{2}"
-PG_JDBC_CONNECTION_STRING_ALT = "jdbc:postgresql://{0}:{1}/{2}"
-
-UBUNTU_PG_HBA_ROOT = "/etc/postgresql"
-PG_HBA_ROOT_DEFAULT = "/var/lib/pgsql/data"
-
-SETUP_DB_CMD = ['su', '-', 'postgres',
- '--command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'" -v dbname="{3}"']
-UPGRADE_STACK_CMD = ['su', 'postgres',
- '--command=psql -f {0} -v stack_name="\'{1}\'" -v stack_version="\'{2}\'" -v dbname="{3}"']
-
-CHANGE_OWNER_COMMAND = ['su', '-', 'postgres',
- '--command=/var/lib/ambari-server/resources/scripts/change_owner.sh -d {0} -s {1} -o {2}']
-
-PG_ERROR_BLOCKED = "is being accessed by other users"
-PG_STATUS_RUNNING = get_running_status()
-PG_DEFAULT_PASSWORD = "bigdata"
-SERVICE_CMD = "/usr/bin/env service"
-PG_SERVICE_NAME = "postgresql"
-PG_HBA_DIR = utils.get_postgre_hba_dir()
-
-PG_ST_CMD = "%s %s status" % (SERVICE_CMD, PG_SERVICE_NAME)
-if os.path.isfile("/usr/bin/postgresql-setup"):
- PG_INITDB_CMD = "/usr/bin/postgresql-setup initdb"
-else:
- PG_INITDB_CMD = "%s %s initdb" % (SERVICE_CMD, PG_SERVICE_NAME)
-
-PG_START_CMD = "%s %s start" % (SERVICE_CMD, PG_SERVICE_NAME)
-PG_RESTART_CMD = "%s %s restart" % (SERVICE_CMD, PG_SERVICE_NAME)
-PG_HBA_RELOAD_CMD = "%s %s reload" % (SERVICE_CMD, PG_SERVICE_NAME)
-
-PG_HBA_CONF_FILE = os.path.join(PG_HBA_DIR, "pg_hba.conf")
-PG_HBA_CONF_FILE_BACKUP = os.path.join(PG_HBA_DIR, "pg_hba_bak.conf.old")
-POSTGRESQL_CONF_FILE = os.path.join(PG_HBA_DIR, "postgresql.conf")
-
-
-# Set database properties to default values
-def load_default_db_properties(args):
- args.persistence_type = 'local'
- args.dbms = DATABASE_NAMES[DATABASE_INDEX]
- args.database_host = "localhost"
- args.database_port = DATABASE_PORTS[DATABASE_INDEX]
- args.database_name = DEFAULT_DB_NAME
- args.database_username = "ambari"
- args.database_password = "bigdata"
- args.sid_or_sname = "sname"
- pass
-
-def configure_database_password(showDefault=True):
- passwordDefault = PG_DEFAULT_PASSWORD
- if showDefault:
- passwordPrompt = 'Enter Database Password (' + passwordDefault + '): '
- else:
- passwordPrompt = 'Enter Database Password: '
- passwordPattern = "^[a-zA-Z0-9_-]*$"
- passwordDescr = "Invalid characters in password. Use only alphanumeric or "\
- "_ or - characters"
-
- password = read_password(passwordDefault, passwordPattern, passwordPrompt,
- passwordDescr)
-
- return password
-
-# Ask user for database connection properties
-def prompt_linux_db_properties(args):
- global DATABASE_INDEX
-
- if args.must_set_database_options:
- load_default_db_properties(args)
- ok = get_YN_input("Enter advanced database configuration [y/n] (n)? ", False)
- if ok:
-
- print "=============================================================================="
- print "Choose one of the following options:"
-
- database_num = str(DATABASE_INDEX + 1)
- database_num = get_validated_string_input(
- "[1] - PostgreSQL (Embedded)\n[2] - Oracle\n[3] - MySQL\n[4] - PostgreSQL\n"
- "==============================================================================\n"
- "Enter choice (" + database_num + "): ",
- database_num,
- "^[1234]$",
- "Invalid number.",
- False
- )
-
- if int(database_num) == 1:
- args.persistence_type = 'local'
- args.database_index = 0
- else:
- args.persistence_type = 'remote'
- selected_db_option = int(database_num)
-
- if selected_db_option == 2:
- args.database_index = 1
- elif selected_db_option == 3:
- args.database_index = 2
- elif selected_db_option == 4:
- args.database_index = 0
- else:
- print_info_msg('Unknown db option, default to embbeded postgres.')
- args.database_index = 0
- pass
- pass
-
- DATABASE_INDEX = args.database_index
- args.dbms = DATABASE_NAMES[args.database_index]
-
- if args.persistence_type != 'local':
- args.database_host = get_validated_string_input(
- "Hostname (" + args.database_host + "): ",
- args.database_host,
+ #Init the database configuration data here, if any
+ self.dbms_full_name = ""
+ self.driver_file_name = ""
+
+ # The values from options supersede the values from properties
+ self.database_host = DBMSConfig._init_member_with_prop_default(options, "database_host",
+ properties, JDBC_HOSTNAME_PROPERTY, "localhost")
+ #self.database_port is set in the subclasses
+ self.database_name = DBMSConfig._init_member_with_prop_default(options, "database_name",
+ properties, JDBC_DATABASE_NAME_PROPERTY, configDefaults.DEFAULT_DB_NAME)
+
+ self.database_username = DBMSConfig._init_member_with_prop_default(options, "database_username",
+ properties, JDBC_USER_NAME_PROPERTY, "ambari")
+ self.database_password = DBMSConfig._init_member_with_default(options, "database_password", _DEFAULT_PASSWORD)
+ self.password_file = get_value_from_properties(properties, JDBC_PASSWORD_PROPERTY, None)
+
+ self.database_url_pattern = ""
+ self.database_url_pattern_alt = ""
+
+ self.database_storage_name = ""
+ self.sid_or_sname = "sname"
+
+ self.init_script_file = ""
+ self.drop_tables_script_file = ""
+ self.client_tool_usage_pattern = ""
+
+ self.jdbc_extra_params = []
+
+ def _prompt_db_properties(self):
+ if not self.silent == True:
+ if self.persistence_type != 'local':
+ self.database_host = get_validated_string_input(
+ "Hostname (" + self.database_host + "): ",
+ self.database_host,
"^[a-zA-Z0-9.\-]*$",
"Invalid hostname.",
False
)
- args.database_port = DATABASE_PORTS[DATABASE_INDEX]
- args.database_port = get_validated_string_input(
- "Port (" + args.database_port + "): ",
- args.database_port,
+ self.database_port = get_validated_string_input(
+ "Port (" + self.database_port + "): ",
+ self.database_port,
"^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$",
"Invalid port.",
False
)
- if args.dbms == "oracle":
- # Oracle uses service name or service id
- idType = "1"
- idType = get_validated_string_input(
- "Select Oracle identifier type:\n1 - " + ORACLE_DB_ID_TYPES[0] +
- "\n2 - " + ORACLE_DB_ID_TYPES[1] + "\n(" + idType + "): ",
- idType,
- "^[12]$",
- "Invalid number.",
- False
- )
-
- if idType == "2":
- args.sid_or_sname = "sid"
-
- IDTYPE_INDEX = int(idType) - 1
- args.database_name = get_validated_service_name(args.database_name,
- IDTYPE_INDEX)
- elif args.dbms in ["mysql", "postgres"]:
- args.database_name = get_validated_db_name(args.database_name)
-
- else:
- # other DB types
- pass
- pass
- else:
- args.database_host = "localhost"
- args.database_port = DATABASE_PORTS[DATABASE_INDEX]
-
- args.database_name = get_validated_db_name(args.database_name)
- pass
+ if not self._configure_database_name():
+ return False
# Username is common for Oracle/MySQL/Postgres
- args.database_username = get_validated_string_input(
- 'Username (' + args.database_username + '): ',
- args.database_username,
+ self.database_username = get_validated_string_input(
+ 'Username (' + self.database_username + '): ',
+ self.database_username,
USERNAME_PATTERN,
"Invalid characters in username. Start with _ or alpha "
"followed by alphanumeric or _ or - characters",
False
)
- args.database_password = configure_database_password(True)
+ self.database_password = LinuxDBMSConfig._configure_database_password(True)
- print_info_msg('Using database options: {database},{host},{port},{schema},{user},{password}'.format(
- database=args.dbms,
- host=args.database_host,
- port=args.database_port,
- schema=args.database_name,
- user=args.database_username,
- password=args.database_password
- ))
+ self._display_db_properties()
+ return True
-# PostgreSQL configuration and setup
-class PGConfig(DBMSConfig):
- def __init__(self):
- #Init the database configuration data here, if any
- pass
+ # Supporting remote server for all the DB types. Supporting local server only for PostgreSQL.
+ def _setup_remote_server(self, args):
+ self._store_remote_properties(args)
+
+ def _setup_remote_database(self):
+ properties = get_ambari_properties()
+ if properties == -1:
+ err = 'Error getting ambari properties'
+ print_error_msg(err)
+ raise FatalException(-1, err)
+
+ if self.ensure_jdbc_driver_installed(properties):
+ print 'Configuring remote database connection properties...'
+ retcode = self._setup_remote_db()
+ if retcode == -1:
+ err = "Remote database setup aborted."
+ raise NonFatalException(err)
+ if not retcode == 0:
+ err = 'Error while configuring connection properties. Exiting'
+ raise FatalException(retcode, err)
+
+ def _reset_remote_database(self):
+ client_usage_cmd_drop = self.client_tool_usage_pattern.format(self.drop_tables_script_file, self.database_username,
+ BLIND_PASSWORD, self.database_name)
+ client_usage_cmd_init = self.client_tool_usage_pattern.format(self.init_script_file, self.database_username,
+ BLIND_PASSWORD, self.database_name)
+
+ print_warning_msg('To reset Ambari Server schema ' +
+ 'you must run the following DDL against the database to '
+ + 'drop the schema:' + os.linesep + client_usage_cmd_drop
+ + os.linesep + 'Then you must run the following DDL ' +
+ 'against the database to create the schema: ' + os.linesep +
+ client_usage_cmd_init + os.linesep)
+
+ def _install_jdbc_driver(self, properties, files_list):
+ if type(files_list) is not int:
+ print 'Copying JDBC drivers to server resources...'
+ try:
+ resources_dir = properties[RESOURCES_DIR_PROPERTY]
+ except KeyError:
+ print_error_msg("There is no value for " + RESOURCES_DIR_PROPERTY + "in " + AMBARI_PROPERTIES_FILE)
+ return False
+
+ db_name = self.dbms_full_name.lower()
+ jdbc_symlink = os.path.join(resources_dir, db_name + "-jdbc-driver.jar")
+ db_default_driver_path = os.path.join(configDefaults.JAVA_SHARE_PATH, self.driver_file_name)
+
+ if os.path.lexists(jdbc_symlink):
+ os.remove(jdbc_symlink)
+
+ copy_status = copy_files(files_list, resources_dir)
- def configure_database_password(showDefault=True):
- passwordDefault = PG_DEFAULT_PASSWORD
+ if not copy_status == 0:
+ raise FatalException(-1, "Failed to copy JDBC drivers to server resources")
+
+ if db_default_driver_path in files_list:
+ os.symlink(os.path.join(resources_dir, self.driver_file_name), jdbc_symlink)
+ else:
+ if files_list != -1:
+ return False
+ return True
+
+ def _configure_database_name(self):
+ return True
+
+ @staticmethod
+ def _configure_database_password(showDefault=True):
+ passwordDefault = _DEFAULT_PASSWORD
if showDefault:
passwordPrompt = 'Enter Database Password (' + passwordDefault + '): '
else:
@@ -228,229 +201,334 @@ class PGConfig(DBMSConfig):
return password
- #
- # Private implementation
- #
- def _change_db_files_owner(args):
- print 'Fixing database objects owner'
- database_name = args.database_name
- new_owner = args.database_username
- if '"' not in new_owner:
- #wrap to allow old username "ambari-server", postgres only
- new_owner = '\'"{0}"\''.format(new_owner)
- pass
+ @staticmethod
+ def _get_validated_db_name(database_storage_name, database_name):
+ return get_validated_string_input(
+ database_storage_name + " Name ("
+ + database_name + "): ",
+ database_name,
+ ".*",
+ "Invalid " + database_storage_name + " name.",
+ False
+ )
+
+ def _display_db_properties(self):
+ print_info_msg('Using database options: {database},{host},{port},{schema},{user},{password}'.format(
+ database=self.dbms,
+ host=self.database_host,
+ port=self.database_port,
+ schema=self.database_name,
+ user=self.database_username,
+ password=self.database_password
+ ))
- command = CHANGE_OWNER_COMMAND[:]
- command[-1] = command[-1].format(database_name, 'ambari', new_owner)
- return run_os_command(command)
+ #Check if required jdbc drivers present
+ @staticmethod
+ def _find_jdbc_driver(jdbc_pattern):
+ drivers = []
+ drivers.extend(glob.glob(configDefaults.JAVA_SHARE_PATH + os.sep + jdbc_pattern))
+ if drivers:
+ return drivers
+ return -1
- def _configure_pg_hba_ambaridb_users(self):
- args = optparse.Values()
- configure_database_username_password(args)
+ # Let the console user initialize the remote database schema
+ def _setup_remote_db(self):
+ setup_msg = "Before starting Ambari Server, you must run the following DDL " \
+ "against the database to create the schema: {0}".format(self.init_script_file)
- with open(PG_HBA_CONF_FILE, "a") as pgHbaConf:
- pgHbaConf.write("\n")
- pgHbaConf.write("local all " + args.database_username +
- ",mapred md5")
- pgHbaConf.write("\n")
- pgHbaConf.write("host all " + args.database_username +
- ",mapred 0.0.0.0/0 md5")
- pgHbaConf.write("\n")
- pgHbaConf.write("host all " + args.database_username +
- ",mapred ::/0 md5")
- pgHbaConf.write("\n")
- retcode, out, err = run_os_command(PG_HBA_RELOAD_CMD)
- if not retcode == 0:
- raise FatalException(retcode, err)
+ print_warning_msg(setup_msg)
- def _configure_pg_hba_postgres_user(self):
- postgresString = "all postgres"
- for line in fileinput.input(PG_HBA_CONF_FILE, inplace=1):
- print re.sub('all\s*all', postgresString, line),
- os.chmod(PG_HBA_CONF_FILE, 0644)
+ proceed = get_YN_input("Proceed with configuring remote database connection properties [y/n] (y)? ", True)
+ retCode = 0 if proceed else -1
- def _configure_postgresql_conf(self):
- listenAddress = "listen_addresses = '*' #"
- for line in fileinput.input(POSTGRESQL_CONF_FILE, inplace=1):
- print re.sub('#+listen_addresses.*?(#|$)', listenAddress, line),
- os.chmod(POSTGRESQL_CONF_FILE, 0644)
+ return retCode
# Store set of properties for remote database connection
- def _store_remote_properties(args):
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return -1
-
- isSecure = get_is_secure(properties)
-
- properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "remote")
+ def _store_remote_properties(self, properties):
+ properties.process_pair(PERSISTENCE_TYPE_PROPERTY, self.persistence_type)
- properties.process_pair(JDBC_DATABASE_PROPERTY, args.dbms)
- properties.process_pair(JDBC_HOSTNAME_PROPERTY, args.database_host)
- properties.process_pair(JDBC_PORT_PROPERTY, args.database_port)
- properties.process_pair(JDBC_SCHEMA_PROPERTY, args.database_name)
+ properties.process_pair(JDBC_DATABASE_PROPERTY, self.dbms)
+ properties.process_pair(JDBC_HOSTNAME_PROPERTY, self.database_host)
+ properties.process_pair(JDBC_PORT_PROPERTY, self.database_port)
+ properties.process_pair(JDBC_DATABASE_NAME_PROPERTY, self.database_name)
- properties.process_pair(JDBC_DRIVER_PROPERTY, DBCN.get_driver_name())
+ properties.process_pair(JDBC_DRIVER_PROPERTY, self.driver_class_name)
# fully qualify the hostname to make sure all the other hosts can connect
# to the jdbc hostname since its passed onto the agents for RCA
- jdbc_hostname = args.database_host
- if (args.database_host == "localhost"):
+ jdbc_hostname = self.database_host
+ if (self.database_host == "localhost"):
jdbc_hostname = socket.getfqdn()
- #TODO: Implement the DBCN connection string generation
- #connectionStringFormat = DATABASE_CONNECTION_STRINGS
- #if args.sid_or_sname == "sid":
- # connectionStringFormat = DATABASE_CONNECTION_STRINGS_ALT
- #properties.process_pair(JDBC_URL_PROPERTY, connectionStringFormat[DATABASE_INDEX].format(jdbc_hostname, args.database_port, args.database_name))
- properties.process_pair(JDBC_URL_PROPERTY, DBCN.get_connection_string())
- properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username)
+ connectionStringFormat = self.database_url_pattern
+ if self.sid_or_sname == "sid":
+ connectionStringFormat = self.database_url_pattern_alt
+ properties.process_pair(JDBC_URL_PROPERTY, connectionStringFormat.format(jdbc_hostname, self.database_port, self.database_name))
+ properties.process_pair(JDBC_USER_NAME_PROPERTY, self.database_username)
properties.process_pair(JDBC_PASSWORD_PROPERTY,
- store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
+ store_password_file(self.database_password, JDBC_PASSWORD_FILENAME))
# save any other defined properties to pass to JDBC
- if DATABASE_INDEX < len(DATABASE_JDBC_PROPERTIES):
- for pair in DATABASE_JDBC_PROPERTIES[DATABASE_INDEX]:
- properties.process_pair(JDBC_PROPERTIES_PREFIX + pair[0], pair[1])
+ for pair in self.jdbc_extra_params:
+ properties.process_pair(JDBC_PROPERTIES_PREFIX + pair[0], pair[1])
- if isSecure:
- encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password)
- if encrypted_password != args.database_password:
+ if self.isSecure:
+ encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, self.database_password)
+ if encrypted_password != self.database_password:
properties.process_pair(JDBC_PASSWORD_PROPERTY, encrypted_password)
pass
- properties.process_pair(JDBC_RCA_DRIVER_PROPERTY, DBCN.get_driver_name())
- properties.process_pair(JDBC_RCA_URL_PROPERTY, DBCN.get_connection_string())
- properties.process_pair(JDBC_RCA_USER_NAME_PROPERTY, args.database_username)
+ properties.process_pair(JDBC_RCA_DRIVER_PROPERTY, self.driver_class_name)
+ properties.process_pair(JDBC_RCA_URL_PROPERTY, self.database_url_pattern.format(jdbc_hostname, self.database_port, self.database_name))
+ properties.process_pair(JDBC_RCA_USER_NAME_PROPERTY, self.database_username)
properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY,
- store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
- if isSecure:
- encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password)
- if encrypted_password != args.database_password:
+ store_password_file(self.database_password, JDBC_PASSWORD_FILENAME))
+ if self.isSecure:
+ encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, self.database_password)
+ if encrypted_password != self.database_password:
properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY, encrypted_password)
- pass
-
- conf_file = properties.fileName
- try:
- properties.store(open(conf_file, "w"))
- except Exception, e:
- print 'Could not write ambari config file "%s": %s' % (conf_file, e)
- return -1
- return 0
+# PostgreSQL configuration and setup
+class PGConfig(LinuxDBMSConfig):
+ # PostgreSQL settings
+ SETUP_DB_CMD = ['su', '-', 'postgres',
+ '--command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'" -v dbname="{3}"']
+ UPGRADE_STACK_CMD = ['su', 'postgres',
+ '--command=psql -f {0} -v stack_name="\'{1}\'" -v stack_version="\'{2}\'" -v dbname="{3}"']
+
+ CHANGE_OWNER_COMMAND = ['su', '-', 'postgres',
+ '--command=/var/lib/ambari-server/resources/scripts/change_owner.sh -d {0} -s {1} -o {2}']
+
+ PG_ERROR_BLOCKED = "is being accessed by other users"
+ PG_STATUS_RUNNING = get_postgre_running_status(OS_TYPE)
+ SERVICE_CMD = "/usr/bin/env service"
+ PG_SERVICE_NAME = "postgresql"
+ PG_HBA_DIR = get_postgre_hba_dir(OSCheck.get_os_family())
+
+ PG_ST_CMD = "%s %s status" % (SERVICE_CMD, PG_SERVICE_NAME)
+ if os.path.isfile("/usr/bin/postgresql-setup"):
+ PG_INITDB_CMD = "/usr/bin/postgresql-setup initdb"
+ else:
+ PG_INITDB_CMD = "%s %s initdb" % (SERVICE_CMD, PG_SERVICE_NAME)
- #
- # Public methods
- #
- def configure_postgres(self):
- if os.path.isfile(PG_HBA_CONF_FILE):
- if not os.path.isfile(PG_HBA_CONF_FILE_BACKUP):
- shutil.copyfile(PG_HBA_CONF_FILE, PG_HBA_CONF_FILE_BACKUP)
- else:
- #Postgres has been configured before, must not override backup
- print "Backup for pg_hba found, reconfiguration not required"
- return 0
- self._configure_pg_hba_postgres_user()
- self._configure_pg_hba_ambaridb_users()
- os.chmod(PG_HBA_CONF_FILE, 0644)
- self._configure_postgresql_conf()
- #restart postgresql if already running
- pg_status = get_postgre_status()
- if pg_status == PG_STATUS_RUNNING:
- retcode = restart_postgres()
- return retcode
- return 0
+ PG_START_CMD = "%s %s start" % (SERVICE_CMD, PG_SERVICE_NAME)
+ PG_RESTART_CMD = "%s %s restart" % (SERVICE_CMD, PG_SERVICE_NAME)
+ PG_HBA_RELOAD_CMD = "%s %s reload" % (SERVICE_CMD, PG_SERVICE_NAME)
- def configure_database(self, args):
- prompt_db_properties(args)
+ PG_HBA_CONF_FILE = os.path.join(PG_HBA_DIR, "pg_hba.conf")
+ PG_HBA_CONF_FILE_BACKUP = os.path.join(PG_HBA_DIR, "pg_hba_bak.conf.old")
+ POSTGRESQL_CONF_FILE = os.path.join(PG_HBA_DIR, "postgresql.conf")
+ def __init__(self, options, properties, storage_type):
+ super(PGConfig, self).__init__(options, properties, storage_type)
- #DB setup should be done last after doing any setup.
+ #Init the database configuration data here, if any
+ self.dbms = "postgres"
+ self.dbms_full_name = "PostgreSQL"
+ self.driver_class_name = "org.postgresql.Driver"
+ self.driver_file_name = "postgresql-jdbc.jar"
- if is_local_database(args):
- #check if jdbc user is changed
- is_user_changed = is_jdbc_user_changed(args)
+ self.database_storage_name = "Database"
- print 'Default properties detected. Using built-in database.'
- store_local_properties(args)
+ # PostgreSQL seems to require additional schema coordinates
+ self.postgres_schema = get_value_from_properties(properties, JDBC_POSTGRES_SCHEMA_PROPERTY, self.database_name)
+ self.database_port = DBMSConfig._init_member_with_prop_default(options, "database_port",
+ properties, JDBC_PORT_PROPERTY, "5432")
- print 'Checking PostgreSQL...'
- retcode = check_postgre_up()
- if not retcode == 0:
- err = 'Unable to start PostgreSQL server. Exiting'
- raise FatalException(retcode, err)
+ self.database_url_pattern = "jdbc:postgresql://{0}:{1}/{2}"
+ self.database_url_pattern_alt = "jdbc:postgresql://{0}:{1}/{2}"
- print 'Configuring local database...'
- retcode, outdata, errdata = setup_db(args)
- if not retcode == 0:
- err = 'Running database init script was failed. Exiting.'
- raise FatalException(retcode, err)
+ self.JDBC_DRIVER_INSTALL_MSG = 'Before starting Ambari Server, ' \
+ 'you must copy the {0} JDBC driver JAR file to {1}.'.format(
+ self.dbms_full_name, configDefaults.JAVA_SHARE_PATH)
- if is_user_changed:
- #remove backup for pg_hba in order to reconfigure postgres
- remove_file(PG_HBA_CONF_FILE_BACKUP)
+ self._is_user_changed = False
- print 'Configuring PostgreSQL...'
- retcode = configure_postgres()
- if not retcode == 0:
- err = 'Unable to configure PostgreSQL server. Exiting'
- raise FatalException(retcode, err)
+ self.init_script_file = "/var/lib/ambari-server/resources/Ambari-DDL-Postgres-CREATE.sql"
+ self.drop_tables_script_file = "/var/lib/ambari-server/resources/Ambari-DDL-Postgres-DROP.sql"
+ self.client_tool_usage_pattern = 'su -postgres --command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'"'
- else:
- retcode = self._store_remote_properties(args)
- if retcode != 0:
- err = 'Unable to save config file'
- raise FatalException(retcode, err)
+ #
+ # Public methods
+ #
- check_jdbc_drivers(args)
+ #
+ # Private implementation
+ #
+ # Supporting remote server for all the DB types. Supporting local server only for PostgreSQL.
+ def _setup_local_server(self, properties):
+ # check if jdbc user is changed
+ self._is_user_changed = PGConfig._is_jdbc_user_changed(self.database_username)
+ print 'Default properties detected. Using built-in database.'
+ self._store_local_properties(properties)
+
+ def _setup_local_database(self):
+ print 'Checking PostgreSQL...'
+ (pg_status, retcode, out, err) = self._check_postgre_up()
+ if not retcode == 0:
+ err = 'Unable to start PostgreSQL server. Exiting'
+ raise FatalException(retcode, err)
+ print 'Configuring local database...'
+ retcode, out, err = self._setup_db()
+ if not retcode == 0:
+ err = 'Running database init script was failed. Exiting.'
+ raise FatalException(retcode, err)
+ if self._is_user_changed:
+ #remove backup for pg_hba in order to reconfigure postgres
+ remove_file(PGConfig.PG_HBA_CONF_FILE_BACKUP)
+ print 'Configuring PostgreSQL...'
+ retcode, out, err = self._configure_postgres()
+ if not retcode == 0:
+ err = 'Unable to configure PostgreSQL server. Exiting'
+ raise FatalException(retcode, err)
- print 'Configuring remote database connection properties...'
- retcode = setup_remote_db(args)
- if retcode == -1:
- err = "Remote database setup aborted."
- raise NonFatalException(err)
+ def _reset_local_database(self):
+ dbname = self.database_name
+ filename = self.drop_script_file
+ username = self.database_username
+ password = self.database_password
+ command = PGConfig.SETUP_DB_CMD[:]
+ command[-1] = command[-1].format(filename, username, password, dbname)
+ drop_retcode, drop_outdata, drop_errdata = run_os_command(command)
+ if not drop_retcode == 0:
+ raise FatalException(1, drop_errdata)
+ if drop_errdata and PGConfig.PG_ERROR_BLOCKED in drop_errdata:
+ raise FatalException(1, "Database is in use. Please, make sure all connections to the database are closed")
+ if drop_errdata and VERBOSE:
+ print_warning_msg(drop_errdata)
+ print_info_msg("About to run database setup")
+ retcode, outdata, errdata = self._setup_db()
+ if errdata and VERBOSE:
+ print_warning_msg(errdata)
+ if (errdata and 'ERROR' in errdata.upper()) or (drop_errdata and 'ERROR' in drop_errdata.upper()):
+ if not VERBOSE:
+ raise NonFatalException("Non critical error in DDL, use --verbose for more information")
+ else:
+ raise NonFatalException("Non critical error in DDL")
- if not retcode == 0:
- err = 'Error while configuring connection properties. Exiting'
- raise FatalException(retcode, err)
- check_jdbc_drivers(args)
+ def _is_jdbc_driver_installed(self, properties):
+ return 0
+ def _configure_database_name(self):
+ self.database_name = LinuxDBMSConfig._get_validated_db_name(self.database_storage_name, self.database_name)
+ self.postgres_schema = self._get_validated_db_schema(self.postgres_schema)
+ return True
- def configure_database_username_password(self, args):
+ def _get_validated_db_schema(self, postgres_schema):
+ return get_validated_string_input(
+ "Postgres schema (" + postgres_schema + "): ",
+ postgres_schema,
+ "^[a-zA-Z0-9_\-]*$",
+ "Invalid schema name.",
+ False, allowEmpty=True
+ )
+
+ # Check if jdbc user is changed
+ @staticmethod
+ def _is_jdbc_user_changed(database_username):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
- return -1
-
- username = properties[JDBC_USER_NAME_PROPERTY]
- passwordProp = properties[JDBC_PASSWORD_PROPERTY]
- dbname = properties[JDBC_DATABASE_PROPERTY]
-
- if username and passwordProp and dbname:
- print_info_msg("Database username + password already configured")
- args.database_username = username
- args.database_name = dbname
- if is_alias_string(passwordProp):
- args.database_password = decrypt_password_for_alias(JDBC_RCA_PASSWORD_ALIAS)
+ return None
+
+ previos_user = get_value_from_properties(properties, JDBC_USER_NAME_PROPERTY, "")
+ new_user = database_username
+
+ if previos_user and new_user:
+ if previos_user != new_user:
+ return True
else:
- if os.path.exists(passwordProp):
- with open(passwordProp, 'r') as file:
- args.database_password = file.read()
+ return False
- return 1
- else:
- print_error_msg("Connection properties not set in config file.")
+ return None
+
+ # Store local database connection properties
+ def _store_local_properties(self, properties):
+ properties.removeOldProp(JDBC_DATABASE_PROPERTY)
+ properties.removeOldProp(JDBC_DATABASE_NAME_PROPERTY)
+ properties.removeOldProp(JDBC_POSTGRES_SCHEMA_PROPERTY)
+ properties.removeOldProp(JDBC_HOSTNAME_PROPERTY)
+ properties.removeOldProp(JDBC_RCA_DRIVER_PROPERTY)
+ properties.removeOldProp(JDBC_RCA_URL_PROPERTY)
+ properties.removeOldProp(JDBC_PORT_PROPERTY)
+ properties.removeOldProp(JDBC_DRIVER_PROPERTY)
+ properties.removeOldProp(JDBC_URL_PROPERTY)
+
+ # Store the properties
+ properties.process_pair(PERSISTENCE_TYPE_PROPERTY, self.persistence_type)
+ properties.process_pair(JDBC_DATABASE_PROPERTY, self.dbms)
+ properties.process_pair(JDBC_DATABASE_NAME_PROPERTY, self.database_name)
+ properties.process_pair(JDBC_POSTGRES_SCHEMA_PROPERTY, self.postgres_schema)
+ properties.process_pair(JDBC_USER_NAME_PROPERTY, self.database_username)
+
+ if self.isSecure:
+ encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, self.database_password)
+ if self.database_password != encrypted_password:
+ properties.process_pair(JDBC_PASSWORD_PROPERTY, encrypted_password)
+
+ password_file = store_password_file(self.database_password, JDBC_PASSWORD_FILENAME)
+ properties.process_pair(JDBC_PASSWORD_PROPERTY, password_file)
+
+ @staticmethod
+ def _get_postgre_status():
+ retcode, out, err = run_os_command(PGConfig.PG_ST_CMD)
+ try:
+ pg_status = re.search('(stopped|running)', out, re.IGNORECASE).group(0).lower()
+ except AttributeError:
+ pg_status = None
+ return pg_status, retcode, out, err
- def setup_db(self, args):
- self.configure_database_username_password(args)
+ def _check_postgre_up(self):
+ pg_status, retcode, out, err = PGConfig._get_postgre_status()
+ if pg_status == PGConfig.PG_STATUS_RUNNING:
+ print_info_msg("PostgreSQL is running")
+ return pg_status, 0, out, err
+ else:
+ # run initdb only on non ubuntu systems as ubuntu does not have initdb cmd.
+ if OS_TYPE != OSConst.OS_UBUNTU:
+ print "Running initdb: This may take upto a minute."
+ retcode, out, err = run_os_command(PGConfig.PG_INITDB_CMD)
+ if retcode == 0:
+ print out
+ print "About to start PostgreSQL"
+ try:
+ process = subprocess.Popen(PGConfig.PG_START_CMD.split(' '),
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ if OS_TYPE == OSConst.OS_SUSE:
+ time.sleep(20)
+ result = process.poll()
+ print_info_msg("Result of postgres start cmd: " + str(result))
+ if result is None:
+ process.kill()
+ pg_status, retcode, out, err = PGConfig._get_postgre_status()
+ else:
+ retcode = result
+ else:
+ out, err = process.communicate()
+ retcode = process.returncode
+ pg_status, retcode, out, err = PGConfig._get_postgre_status()
+ if pg_status == PGConfig.PG_STATUS_RUNNING:
+ print_info_msg("Postgres process is running. Returning...")
+ return pg_status, 0, out, err
+ except (Exception), e:
+ pg_status, retcode, out, err = PGConfig._get_postgre_status()
+ if pg_status == PGConfig.PG_STATUS_RUNNING:
+ return pg_status, 0, out, err
+ else:
+ print_error_msg("Postgres start failed. " + str(e))
+ return pg_status, retcode, out, err
- dbname = args.database_name
- scriptFile = args.init_script_file
- username = args.database_username
- password = args.database_password
+ def _setup_db(self):
+ #password access to ambari-server and mapred
+ dbname = self.database_name
+ scriptFile = self.init_script_file
+ username = self.database_username
+ password = self.database_password
#setup DB
- command = SETUP_DB_CMD[:]
+ command = PGConfig.SETUP_DB_CMD[:]
command[-1] = command[-1].format(scriptFile, username, password, dbname)
for i in range(SETUP_DB_CONNECT_ATTEMPTS):
@@ -466,21 +544,99 @@ class PGConfig(DBMSConfig):
time.sleep(SETUP_DB_CONNECT_TIMEOUT)
print 'unable to connect to database'
- utils.print_error_msg(errdata)
+ print_error_msg(errdata)
return retcode, outdata, errdata
- # Initialize remote database schema
- def setup_remote_db(args):
+ @staticmethod
+ def _configure_pg_hba_ambaridb_users(conf_file, database_username):
+ with open(conf_file, "a") as pgHbaConf:
+ pgHbaConf.write("\n")
+ pgHbaConf.write("local all " + database_username +
+ ",mapred md5")
+ pgHbaConf.write("\n")
+ pgHbaConf.write("host all " + database_username +
+ ",mapred 0.0.0.0/0 md5")
+ pgHbaConf.write("\n")
+ pgHbaConf.write("host all " + database_username +
+ ",mapred ::/0 md5")
+ pgHbaConf.write("\n")
+ retcode, out, err = run_os_command(PGConfig.PG_HBA_RELOAD_CMD)
+ if not retcode == 0:
+ raise FatalException(retcode, err)
- setup_msg = "Before starting Ambari Server, you must run the following DDL " \
- "against the database to create the schema: {0}".format(DATABASE_INIT_SCRIPTS[DATABASE_INDEX])
+ @staticmethod
+ def _configure_pg_hba_postgres_user():
+ postgresString = "all postgres"
+ for line in fileinput.input(PGConfig.PG_HBA_CONF_FILE, inplace=1):
+ print re.sub('all\s*all', postgresString, line),
+ os.chmod(PGConfig.PG_HBA_CONF_FILE, 0644)
- print_warning_msg(setup_msg)
+ @staticmethod
+ def _configure_postgresql_conf():
+ listenAddress = "listen_addresses = '*' #"
+ for line in fileinput.input(PGConfig.POSTGRESQL_CONF_FILE, inplace=1):
+ print re.sub('#+listen_addresses.*?(#|$)', listenAddress, line),
+ os.chmod(PGConfig.POSTGRESQL_CONF_FILE, 0644)
- proceed = get_YN_input("Proceed with configuring remote database connection properties [y/n] (y)? ", True)
- retCode = 0 if proceed else -1
+ def _configure_postgres(self):
+ if os.path.isfile(PGConfig.PG_HBA_CONF_FILE):
+ if not os.path.isfile(PGConfig.PG_HBA_CONF_FILE_BACKUP):
+ shutil.copyfile(PGConfig.PG_HBA_CONF_FILE, PGConfig.PG_HBA_CONF_FILE_BACKUP)
+ else:
+ #Postgres has been configured before, must not override backup
+ print "Backup for pg_hba found, reconfiguration not required"
+ return 0, "", ""
+ PGConfig._configure_pg_hba_postgres_user()
+ PGConfig._configure_pg_hba_ambaridb_users(PGConfig.PG_HBA_CONF_FILE, self.database_username)
+ os.chmod(PGConfig.PG_HBA_CONF_FILE, 0644)
+ PGConfig._configure_postgresql_conf()
+ #restart postgresql if already running
+ pg_status, retcode, out, err = PGConfig._get_postgre_status()
+ if pg_status == PGConfig.PG_STATUS_RUNNING:
+ retcode, out, err = PGConfig._restart_postgres()
+ return retcode, out, err
+ return 0, "", ""
+
+ @staticmethod
+ def _restart_postgres():
+ print "Restarting PostgreSQL"
+ process = subprocess.Popen(PGConfig.PG_RESTART_CMD.split(' '),
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ time.sleep(5)
+ result = process.poll()
+ if result is None:
+ print_info_msg("Killing restart PostgresSQL process")
+ process.kill()
+ pg_status, retcode, out, err = PGConfig._get_postgre_status()
+ # SUSE linux set status of stopped postgresql proc to unused
+ if pg_status == "unused" or pg_status == "stopped":
+ print_info_msg("PostgreSQL is stopped. Restarting ...")
+ retcode, out, err = run_os_command(PGConfig.PG_START_CMD)
+ return retcode, out, err
+ return 0, "", ""
+
+ def _store_remote_properties(self, properties):
+ super(PGConfig, self)._store_remote_properties(properties)
+
+ properties.process_pair(JDBC_POSTGRES_SCHEMA_PROPERTY, self.postgres_schema)
+
+
+ def _change_db_files_owner(args):
+ print 'Fixing database objects owner'
+ database_name = args.database_name
+ new_owner = args.database_username
+ if '"' not in new_owner:
+ #wrap to allow old username "ambari-server", postgres only
+ new_owner = '\'"{0}"\''.format(new_owner)
+ pass
+
+ command = PGConfig.CHANGE_OWNER_COMMAND[:]
+ command[-1] = command[-1].format(database_name, 'ambari', new_owner)
+ return run_os_command(command)
- return retCode
def change_db_files_owner(self, args):
if args.persistence_type == 'local':
@@ -488,253 +644,120 @@ class PGConfig(DBMSConfig):
if not retcode == 0:
raise FatalException(20, 'Unable to change owner of database objects')
- def reset_remote_db(self, args):
- client_usage_cmd_drop = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_DROP_SCRIPTS[DATABASE_INDEX], args.database_username,
- BLIND_PASSWORD, args.database_name)
- client_usage_cmd_init = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_INIT_SCRIPTS[DATABASE_INDEX], args.database_username,
- BLIND_PASSWORD, args.database_name)
+def createPGConfig(options, properties, storage_type, dbId):
+ return PGConfig(options, properties, storage_type)
- print_warning_msg('To reset Ambari Server schema ' +
- 'you must run the following DDL against the database to '
- + 'drop the schema:' + os.linesep + client_usage_cmd_drop
- + os.linesep + 'Then you must run the following DDL ' +
- 'against the database to create the schema: ' + os.linesep +
- client_usage_cmd_init + os.linesep)
- def reset_local_db(args):
- dbname = args.database_name
- filename = args.drop_script_file
- username = args.database_username
- password = args.database_password
- command = SETUP_DB_CMD[:]
- command[-1] = command[-1].format(filename, username, password, dbname)
- drop_retcode, drop_outdata, drop_errdata = run_os_command(command)
- if not drop_retcode == 0:
- raise FatalException(1, drop_errdata)
- if drop_errdata and PG_ERROR_BLOCKED in drop_errdata:
- raise FatalException(1, "Database is in use. Please, make sure all connections to the database are closed")
- if drop_errdata and VERBOSE:
- print_warning_msg(drop_errdata)
- print_info_msg("About to run database setup")
- retcode, outdata, errdata = setup_db(args)
- if errdata and VERBOSE:
- print_warning_msg(errdata)
- if (errdata and 'ERROR' in errdata.upper()) or (drop_errdata and 'ERROR' in drop_errdata.upper()):
- if not VERBOSE:
- raise NonFatalException("Non critical error in DDL, use --verbose for more information")
- else:
- raise NonFatalException("Non critical error in DDL")
+class OracleConfig(LinuxDBMSConfig):
+ def __init__(self, options, properties, storage_type):
+ super(OracleConfig, self).__init__(options, properties, storage_type)
-# PostgreSQL database
-class PGDatabase:
- _driverName = ''
- _connectionString = ''
+ #Init the database configuration data here, if any
+ self.dbms = "oracle"
+ self.dbms_full_name = "Oracle"
+ self.driver_class_name = "oracle.jdbc.driver.OracleDriver"
+ self.driver_file_name = "ojdbc6.jar"
- def __init__(self):
- #Init the database connection here, if any
- pass
+ self.database_storage_name = "Service"
- #
- # Private implementation
- #
+ self.database_port = DBMSConfig._init_member_with_prop_default(options, "database_port",
+ properties, JDBC_PORT_PROPERTY, "1521")
- # Get database client executable path
- def get_db_cli_tool(self, args):
- for tool in DATABASE_CLI_TOOLS[DATABASE_INDEX]:
- cmd = CHECK_COMMAND_EXIST_CMD.format(tool)
- ret, out, err = run_in_shell(cmd)
- if ret == 0:
- return get_exec_path(tool)
+ self.database_url_pattern = "jdbc:oracle:thin:@{0}:{1}/{2}"
+ self.database_url_pattern_alt = "jdbc:oracle:thin:@{0}:{1}:{2}"
- return None
+ self.JDBC_DRIVER_INSTALL_MSG = 'Before starting Ambari Server, ' \
+ 'you must copy the {0} JDBC driver JAR file to {1}.'.format(
+ self.dbms_full_name, configDefaults.JAVA_SHARE_PATH)
+
+ self.init_script_file = "/var/lib/ambari-server/resources/Ambari-DDL-Oracle-CREATE.sql'"
+ self.drop_tables_script_file = "/var/lib/ambari-server/resources/Ambari-DDL-Oracle-DROP.sql"
+ self.client_tool_usage_pattern = 'sqlplus {1}/{2} < {0}'
+
+ self.jdbc_extra_params = [
+ ["oracle.net.CONNECT_TIMEOUT", "2000"], # socket level timeout
+ ["oracle.net.READ_TIMEOUT", "2000"], # socket level timeout
+ ["oracle.jdbc.ReadTimeout", "8000"] # query fetch timeout
+ ]
#
- # Public interface
+ # Private implementation
#
- def get_driver_name(self):
- return self._driverName
+ def _is_jdbc_driver_installed(self, properties):
+ return LinuxDBMSConfig._find_jdbc_driver("*ojdbc*.jar")
+
+ def _configure_database_name(self):
+ if self.persistence_type != 'local':
+ # Oracle uses service name or service id
+ idType = "1"
+ idType = get_validated_string_input(
+ "Select Oracle identifier type:\n1 - " + ORACLE_DB_ID_TYPES[0] +
+ "\n2 - " + ORACLE_DB_ID_TYPES[1] + "\n(" + idType + "): ",
+ idType,
+ "^[12]$",
+ "Invalid number.",
+ False
+ )
- def get_connection_string(self):
- return self._connectionString
+ if idType == "2":
+ self.sid_or_sname = "sid"
- def connect(self, args):
- if args.persistence_type == "local":
- return self.check_postgre_up()
+ IDTYPE_INDEX = int(idType) - 1
+ self.database_name = OracleConfig._get_validated_service_name(self.database_name,
+ IDTYPE_INDEX)
else:
- return 0
+ self.database_name = LinuxDBMSConfig._get_validated_db_name(self.database_storage_name, self.database_name)
- def get_running_status(self):
- """Return postgre running status indicator"""
- if OS_TYPE == OSConst.OS_UBUNTU:
- return "%s/main" % PGDatabase.get_ubuntu_db_version()
- else:
- return DB_STATUS_RUNNING_DEFAULT
+ return True
@staticmethod
- def get_hba_dir():
- """Return postgre hba dir location depends on OS"""
- if OS_TYPE == OSConst.OS_UBUNTU:
- return "%s/%s/main" % (UBUNTU_PG_HBA_ROOT, PGDatabase.get_ubuntu_db_version())
- else:
- return PG_HBA_ROOT_DEFAULT
-
- @staticmethod
- def get_ubuntu_db_version():
- """Return installed version of postgre server. In case of several
- installed versions will be returned a more new one.
- """
- postgre_ver = ""
+ def _get_validated_service_name(service_name, index):
+ return get_validated_string_input(
+ ORACLE_DB_ID_TYPES[index] + " (" + service_name + "): ",
+ service_name,
+ ".*",
+ "Invalid " + ORACLE_DB_ID_TYPES[index] + ".",
+ False
+ )
- if os.path.isdir(UBUNTU_PG_HBA_ROOT): # detect actual installed versions of PG and select a more new one
- postgre_ver = sorted(
- [fld for fld in os.listdir(UBUNTU_PG_HBA_ROOT) if os.path.isdir(os.path.join(UBUNTU_PG_HBA_ROOT, fld))], reverse=True)
- if len(postgre_ver) > 0:
- return postgre_ver[0]
- return postgre_ver
+def createOracleConfig(options, properties, storage_type, dbId):
+ return OracleConfig(options, properties, storage_type)
- def restart_postgres():
- print "Restarting PostgreSQL"
- process = subprocess.Popen(PG_RESTART_CMD.split(' '),
- stdout=subprocess.PIPE,
- stdin=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
- time.sleep(5)
- result = process.poll()
- if result is None:
- print_info_msg("Killing restart PostgresSQL process")
- process.kill()
- pg_status = get_postgre_status()
- # SUSE linux set status of stopped postgresql proc to unused
- if pg_status == "unused" or pg_status == "stopped":
- print_info_msg("PostgreSQL is stopped. Restarting ...")
- retcode, out, err = run_os_command(PG_START_CMD)
- return retcode
- return 0
+class MySQLConfig(LinuxDBMSConfig):
+ def __init__(self, options, properties, storage_type):
+ super(MySQLConfig, self).__init__(options, properties, storage_type)
- def execute_db_script(self, args, file):
- #password access to ambari-server and mapred
- configure_database_username_password(args)
- dbname = args.database_name
- username = args.database_username
- password = args.database_password
- command = SETUP_DB_CMD[:]
- command[-1] = command[-1].format(file, username, password, dbname)
- retcode, outdata, errdata = run_os_command(command)
- if not retcode == 0:
- print errdata
- return retcode
-
- def execute_remote_script(self, args, scriptPath):
- print_warning_msg("Deprecated method called.")
- tool = get_db_cli_tool(args)
- if not tool:
- # args.warnings.append('{0} not found. Please, run DDL script manually'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
- if VERBOSE:
- print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
- return -1, "Client wasn't found", "Client wasn't found"
-
- os.environ["PGPASSWORD"] = args.database_password
- retcode, out, err = run_in_shell('{0} {1}'.format(tool, POSTGRES_EXEC_ARGS.format(
- args.database_host,
- args.database_port,
- args.database_name,
- args.database_username,
- scriptPath
- )))
- return retcode, out, err
-
- def check_db_consistency(args, file):
- #password access to ambari-server and mapred
- configure_database_username_password(args)
- dbname = args.database_name
- username = args.database_username
- password = args.database_password
- command = SETUP_DB_CMD[:]
- command[-1] = command[-1].format(file, username, password, dbname)
- retcode, outdata, errdata = run_os_command(command)
- if not retcode == 0:
- print errdata
- return retcode
- else:
- # Assumes that the output is of the form ...\n<count>
- print_info_msg("Parsing output: " + outdata)
- lines = outdata.splitlines()
- if (lines[-1] == '3' or lines[-1] == '0'):
- return 0
- return -1
+ #Init the database configuration data here, if any
+ self.dbms = "mysql"
+ self.dbms_full_name = "MySQL"
+ self.driver_class_name = "com.mysql.jdbc.Driver"
+ self.driver_file_name = "mysql-connector-java.jar"
+ self.database_storage_name = "Database"
+ self.database_port = DBMSConfig._init_member_with_prop_default(options, "database_port",
+ properties, JDBC_PORT_PROPERTY, "3306")
- def get_postgre_status():
- retcode, out, err = run_os_command(PG_ST_CMD)
- try:
- pg_status = re.search('(stopped|running)', out, re.IGNORECASE).group(0).lower()
- except AttributeError:
- pg_status = None
- return pg_status
+ self.database_url_pattern = "jdbc:mysql://{0}:{1}/{2}"
+ self.database_url_pattern_alt = "jdbc:mysql://{0}:{1}/{2}"
+ self.JDBC_DRIVER_INSTALL_MSG = 'Before starting Ambari Server, ' \
+ 'you must copy the {0} JDBC driver JAR file to {1}.'.format(
+ self.dbms_full_name, configDefaults.JAVA_SHARE_PATH)
- def check_postgre_up():
- pg_status = get_postgre_status()
- if pg_status == PG_STATUS_RUNNING:
- print_info_msg("PostgreSQL is running")
- return 0
- else:
- # run initdb only on non ubuntu systems as ubuntu does not have initdb cmd.
- if OS_TYPE != OSConst.OS_UBUNTU:
- print "Running initdb: This may take upto a minute."
- retcode, out, err = run_os_command(PG_INITDB_CMD)
- if retcode == 0:
- print out
- print "About to start PostgreSQL"
- try:
- process = subprocess.Popen(PG_START_CMD.split(' '),
- stdout=subprocess.PIPE,
- stdin=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
- if OS_TYPE == OSConst.OS_SUSE:
- time.sleep(20)
- result = process.poll()
- print_info_msg("Result of postgres start cmd: " + str(result))
- if result is None:
- process.kill()
- pg_status = get_postgre_status()
- else:
- retcode = result
- else:
- out, err = process.communicate()
- retcode = process.returncode
- if pg_status == PG_STATUS_RUNNING:
- print_info_msg("Postgres process is running. Returning...")
- return 0
- except (Exception), e:
- pg_status = get_postgre_status()
- if pg_status == PG_STATUS_RUNNING:
- return 0
- else:
- print_error_msg("Postgres start failed. " + str(e))
- return 1
- return retcode
+ self.init_script_file = "/var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql"
+ self.drop_tables_script_file = "/var/lib/ambari-server/resources/Ambari-DDL-MySQL-DROP.sql"
+ self.client_tool_usage_pattern = 'mysql --user={1} --password={2} {3}<{0}'
+ #
+ # Private implementation
+ #
+ def _is_jdbc_driver_installed(self, properties):
+ return LinuxDBMSConfig._find_jdbc_driver("*mysql*.jar")
- def get_validated_db_name(database_name):
- return get_validated_string_input(
- DATABASE_STORAGE_NAMES[DATABASE_INDEX] + " Name ("
- + database_name + "): ",
- database_name,
- ".*",
- "Invalid " + DATABASE_STORAGE_NAMES[DATABASE_INDEX] + " name.",
- False
- )
-
+ def _configure_database_name(self):
+ self.database_name = LinuxDBMSConfig._get_validated_db_name(self.database_storage_name, self.database_name)
+ return True
- def get_validated_service_name(service_name, index):
- return get_validated_string_input(
- ORACLE_DB_ID_TYPES[index] + " (" + service_name + "): ",
- service_name,
- ".*",
- "Invalid " + ORACLE_DB_ID_TYPES[index] + ".",
- False
- )
+def createMySQLConfig(options, properties, storage_type, dbId):
+ return MySQLConfig(options, properties, storage_type)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py b/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py
index 647a940..ed986cb 100644
--- a/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py
+++ b/ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py
@@ -18,71 +18,59 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
+import os
import socket
import string
-import win32api
-
-from ambari_commons.exceptions import *
-from ambari_commons.logging_utils import print_warning_msg
-from ambari_commons.os_utils import search_file
-from ambari_commons.os_windows import *
+from ambari_commons.exceptions import FatalException
+from ambari_commons.logging_utils import print_warning_msg, print_info_msg
+from ambari_commons.os_utils import search_file, run_os_command
+from ambari_commons.os_windows import WinServiceController, os_run_os_command
from ambari_commons.str_utils import compress_backslashes, ensure_double_backslashes
-from ambari_server.setupSecurity import SECURITY_IS_ENCRYPTION_ENABLED, encrypt_password, store_password_file
-from serverConfiguration import *
-from dbConfiguration import *
-from userInput import get_validated_string_input
+from ambari_server.dbConfiguration import AMBARI_DATABASE_NAME, DBMSConfig, DbPropKeys, DbAuthenticationKeys
+from ambari_server.serverConfiguration import get_value_from_properties, configDefaults, \
+ JDBC_DATABASE_PROPERTY, JDBC_DATABASE_NAME_PROPERTY, JDBC_RCA_SCHEMA_PROPERTY, \
+ JDBC_HOSTNAME_PROPERTY, JDBC_PORT_PROPERTY, \
+ JDBC_USE_INTEGRATED_AUTH_PROPERTY, JDBC_USER_NAME_PROPERTY, JDBC_PASSWORD_PROPERTY, JDBC_PASSWORD_FILENAME, \
+ JDBC_DRIVER_PROPERTY, JDBC_URL_PROPERTY, \
+ JDBC_RCA_HOSTNAME_PROPERTY, JDBC_RCA_PORT_PROPERTY, \
+ JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY, JDBC_RCA_USER_NAME_PROPERTY, JDBC_RCA_PASSWORD_ALIAS, JDBC_RCA_PASSWORD_FILE_PROPERTY, \
+ JDBC_RCA_DRIVER_PROPERTY, JDBC_RCA_URL_PROPERTY, \
+ JDBC_DRIVER_PATH_PROPERTY, \
+ JDBC_METRICS_DATABASE_PROPERTY, JDBC_METRICS_SCHEMA_PROPERTY, JDBC_METRICS_HOSTNAME_PROPERTY, JDBC_METRICS_PORT_PROPERTY, \
+ JDBC_METRICS_DRIVER_PROPERTY, JDBC_METRICS_URL_PROPERTY, \
+ JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY, JDBC_METRICS_USER_NAME_PROPERTY, JDBC_METRICS_PASSWORD_PROPERTY, \
+ JDBC_METRICS_PASSWORD_ALIAS, JDBC_METRICS_PASSWORD_FILENAME, \
+ PERSISTENCE_TYPE_PROPERTY, METRICS_PERSISTENCE_TYPE_PROPERTY
+from ambari_server.setupSecurity import encrypt_password, store_password_file
+from ambari_server.userInput import get_validated_string_input
-#Import the SQL Server libraries
# SQL Server settings
-DBPATH = 'C:\\Program Files\\Microsoft SQL Server\\MSSQL12.SQLEXPRESS\\MSSQL\\DATA\\'
-# DBPATH = 'C:\\Program Files\\Microsoft SQL Server\\MSSQL10_50.MSSQLSERVER\\MSSQL\\DATA\\'
-
-DATABASE_DBMS = "sqlserver"
-DATABASE_DRIVER_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
-LOCAL_DATABASE_SERVER = "localhost\\SQLEXPRESS"
-AMBARI_DATABASE_NAME = "ambari"
+DATABASE_DBMS_SQLSERVER = "sqlserver"
+DATABASE_DRIVER_NAME_SQLSERVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+DATABASE_SERVER_SQLSERVER_DEFAULT = "localhost\\SQLEXPRESS"
METRICS_DATABASE_NAME = "HadoopMetrics"
-
-class DbPropKeys:
- def __init__(self, i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key):
- self.reset(i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key)
- pass
-
- def reset(self, i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key):
- self.dbms_key = i_dbms_key
- self.driver_key = i_driver_key
- self.server_key = i_server_key
- self.port_key = i_port_key
- self.db_name_key = i_db_name_key
- self.db_url_key = i_db_url_key
- pass
-
-class AuthenticationKeys:
+class SqlServerAuthenticationKeys(DbAuthenticationKeys):
def __init__(self, i_integrated_auth_key, i_user_name_key, i_password_key, i_password_alias, i_password_filename):
- self.reset(i_integrated_auth_key, i_user_name_key, i_password_key, i_password_alias, i_password_filename)
- pass
-
- def reset(self, i_integrated_auth_key, i_user_name_key, i_password_key, i_password_alias, i_password_filename):
self.integrated_auth_key = i_integrated_auth_key
- self.user_name_key = i_user_name_key
- self.password_key = i_password_key
- self.password_alias = i_password_alias
- self.password_filename = i_password_filename
- pass
+ DbAuthenticationKeys.__init__(self, i_user_name_key, i_password_key, i_password_alias, i_password_filename)
+#
# SQL Server configuration and setup
+#
class SQLServerConfig(DBMSConfig):
- def __init__(self, options, properties):
- super(SQLServerConfig, self).__init__(options, properties)
+ def __init__(self, options, properties, storage_type):
+ super(SQLServerConfig, self).__init__(options, properties, storage_type)
"""
#Just load the defaults. The derived classes will be able to modify them later
"""
- self.dbms = DATABASE_DBMS
- self.driver_name = DATABASE_DRIVER_NAME
+ self.dbms = DATABASE_DBMS_SQLSERVER
+ self.driver_class_name = DATABASE_DRIVER_NAME_SQLSERVER
+
+ self.JDBC_DRIVER_INSTALL_MSG = 'Before starting Ambari Server, you must install the SQL Server JDBC driver.'
# The values from options supersede the values from properties
self.database_host = options.database_host if options.database_host is not None and options.database_host is not "" else \
@@ -93,27 +81,25 @@ class SQLServerConfig(DBMSConfig):
else:
self.database_host = compress_backslashes(self.database_host)
except:
- self.database_host = "localhost\\SQLEXPRESS"
+ self.database_host = DATABASE_SERVER_SQLSERVER_DEFAULT
pass
- self.database_port = options.database_port if options.database_port is not None and options.database_port is not "" else \
- properties.get_property(self.dbPropKeys.port_key)
- self.database_name = options.database_name if options.database_name is not None and options.database_name is not "" else \
- properties.get_property(self.dbPropKeys.db_name_key)
+ self.database_port = DBMSConfig._init_member_with_prop_default(options, "database_port",
+ properties, self.dbPropKeys.port_key, "1433")
+ self.database_name = DBMSConfig._init_member_with_prop_default(options, "database_name",
+ properties, self.dbPropKeys.db_name_key, configDefaults.DEFAULT_DB_NAME)
self.use_windows_authentication = options.database_windows_auth if options.database_windows_auth is True else \
properties.get_property(self.dbAuthKeys.integrated_auth_key)
- self.database_username = options.database_username if options.database_username is not None and options.database_username is not "" \
- else properties.get_property(self.dbAuthKeys.user_name_key)
+ self.database_username = DBMSConfig._init_member_with_prop_default(options, "database_username",
+ properties, self.dbAuthKeys.user_name_key, "")
self.database_password = options.database_password if options.database_password is not None and options.database_password is not "" \
else ""
- self.password_file = properties[self.dbAuthKeys.password_key]
+ self.password_file = get_value_from_properties(properties, self.dbAuthKeys.password_key, None)
self.database_url = self._build_sql_server_connection_string()
self.persistence_property = None
- self.db_title = ""
-
self.env_var_db_name = ""
self.env_var_db_log_name = ""
self.env_var_db_owner = ""
@@ -127,40 +113,6 @@ class SQLServerConfig(DBMSConfig):
def _is_local_database(self):
return False
- def _is_jdbc_driver_installed(self, properties):
- """
- #Attempt to load the sqljdbc4.jar and sqljdbc_auth.dll. This will automatically scan the PATH.
- :param None
- :rtype : bool
- """
- paths = "." + os.pathsep + os.environ["PATH"]
-
- # Find the jar by attempting to load it as a resource dll
- driver_path = search_file("sqljdbc4.jar", paths)
- if not driver_path:
- return 0
-
- auth_dll_path = search_file("sqljdbc_auth.dll", paths)
- if not auth_dll_path:
- return 0
-
- try:
- driver_path = properties[JDBC_DRIVER_PATH_PROPERTY]
- if driver_path is None or driver_path is "":
- return 0
- except Exception:
- # No such attribute set
- return 0
-
- return 1
-
- def get_jdbc_driver_path(self):
- paths = "." + os.pathsep + os.environ["PATH"]
-
- # Find the jar by attempting to load it as a resource dll
- driver_path = search_file("sqljdbc4.jar", paths)
- return driver_path
-
def configure_database_password(showDefault=True):
#No password needed, using SQL Server integrated authentication
pass
@@ -171,7 +123,7 @@ class SQLServerConfig(DBMSConfig):
return True
#prompt for SQL Server host and instance name
- hostname_prompt = "SQL Server host and instance for the {} database: ({}) ".format(self.db_title, self.database_host)
+ hostname_prompt = "SQL Server host and instance for the {0} database: ({1}) ".format(self.db_title, self.database_host)
self.database_host = get_validated_string_input(hostname_prompt, self.database_host, None, None, False, True)
#prompt for SQL Server authentication method
@@ -183,7 +135,7 @@ class SQLServerConfig(DBMSConfig):
user_prompt = \
"[1] - Use SQL Server integrated authentication\n[2] - Use username+password authentication\n" \
- "Enter choice ({}): ".format(auth_option_default)
+ "Enter choice ({0}): ".format(auth_option_default)
auth_option = get_validated_string_input(user_prompt,
auth_option_default,
"^[12]$",
@@ -196,12 +148,12 @@ class SQLServerConfig(DBMSConfig):
else:
self.use_windows_authentication = False
- user_prompt = "SQL Server user name for the {} database: ({}) ".format(self.db_title, self.database_username)
+ user_prompt = "SQL Server user name for the {0} database: ({1}) ".format(self.db_title, self.database_username)
username = get_validated_string_input(user_prompt, self.database_username, None, "User name", False,
False)
self.database_username = username
- user_prompt = "SQL Server password for the {} database: ".format(self.db_title)
+ user_prompt = "SQL Server password for the {0} database: ".format(self.db_title)
password = get_validated_string_input(user_prompt, "", None, "Password", True, False)
self.database_password = password
@@ -218,7 +170,7 @@ class SQLServerConfig(DBMSConfig):
properties.process_pair(self.persistence_property, 'remote')
properties.process_pair(self.dbPropKeys.dbms_key, self.dbms)
- properties.process_pair(self.dbPropKeys.driver_key, self.driver_name)
+ properties.process_pair(self.dbPropKeys.driver_key, self.driver_class_name)
properties.process_pair(self.dbPropKeys.server_key, ensure_double_backslashes(self.database_host))
if self.database_port is not None and self.database_port != "":
properties.process_pair(self.dbPropKeys.port_key, self.database_port)
@@ -230,34 +182,46 @@ class SQLServerConfig(DBMSConfig):
pass
def _setup_remote_database(self):
- print 'Populating {} database structure...'.format(self.db_title)
+ print 'Populating {0} database structure...'.format(self.db_title)
self._populate_database_structure()
def _reset_remote_database(self):
- print 'Resetting {} database structure...'.format(self.db_title)
+ print 'Resetting {0} database structure...'.format(self.db_title)
self._populate_database_structure()
- def _prompt_jdbc_driver_install(self, properties):
- result = False
- msg = 'Before starting Ambari Server, you must install the SQL Server JDBC driver.'
+ def _is_jdbc_driver_installed(self, properties):
+ """
+ #Attempt to find the sqljdbc4.jar and sqljdbc_auth.dll by scanning the PATH.
+ :param None
+ :rtype : bool
+ """
+ paths = "." + os.pathsep + os.environ["PATH"]
- if not self.silent:
- print_warning_msg(msg)
- raw_input(PRESS_ENTER_MSG)
- result = self._is_jdbc_driver_installed(properties)
- return (result, msg)
+ # Find the jar by attempting to load it as a resource dll
+ driver_path = search_file("sqljdbc4.jar", paths)
+ if not driver_path:
+ return 0
+
+ auth_dll_path = search_file("sqljdbc_auth.dll", paths)
+ if not auth_dll_path:
+ return 0
- def _install_jdbc_driver(self, options, properties):
try:
driver_path = properties[JDBC_DRIVER_PATH_PROPERTY]
+ if driver_path is None or driver_path is "":
+ return 0
except Exception:
# No such attribute set
- driver_path = None
+ return 0
+ return 1
+
+ def _install_jdbc_driver(self, properties, files_list):
+ driver_path = get_value_from_properties(properties, JDBC_DRIVER_PATH_PROPERTY, None)
if driver_path is None or driver_path == "":
- driver_path = self.get_jdbc_driver_path()
+ driver_path = self._get_jdbc_driver_path()
properties.process_pair(JDBC_DRIVER_PATH_PROPERTY, driver_path)
return True
@@ -293,11 +257,18 @@ class SQLServerConfig(DBMSConfig):
pass
+ def _get_jdbc_driver_path(self):
+ paths = "." + os.pathsep + os.environ["PATH"]
+
+ # Find the jar by attempting to load it as a resource dll
+ driver_path = search_file("sqljdbc4.jar", paths)
+ return driver_path
+
def _build_sql_server_connection_string(self):
- databaseUrl = "jdbc:sqlserver://{}".format(ensure_double_backslashes(self.database_host))
+ databaseUrl = "jdbc:sqlserver://{0}".format(ensure_double_backslashes(self.database_host))
if self.database_port is not None and self.database_port != "":
- databaseUrl += ":{}".format(self.database_port)
- databaseUrl += ";databaseName={}".format(self.database_name)
+ databaseUrl += ":{0}".format(self.database_port)
+ databaseUrl += ";databaseName={0}".format(self.database_name)
if(self.use_windows_authentication):
databaseUrl += ";integratedSecurity=true"
#No need to append the username and password, the Ambari server adds them by itself when connecting to the database
@@ -335,26 +306,28 @@ class SQLServerConfig(DBMSConfig):
@staticmethod
def _execute_db_script(databaseHost, databaseScript):
- dbCmd = 'sqlcmd -S {} -i {}'.format(databaseHost, databaseScript)
+ dbCmd = 'sqlcmd -S {0} -i {1}'.format(databaseHost, databaseScript)
retCode, outData, errData = run_os_command(['cmd', '/C', dbCmd])
if not retCode == 0:
- err = 'Running database create script failed. Error output: {} Output: {} Exiting.'.format(errData, outData)
+ err = 'Running database create script failed. Error output: {0} Output: {1} Exiting.'.format(errData, outData)
raise FatalException(retCode, err)
print_info_msg("sqlcmd output:")
print_info_msg(outData)
pass
+#
# SQL Server Ambari database configuration and setup
+#
class SQLServerAmbariDBConfig(SQLServerConfig):
- def __init__(self, options, properties):
+ def __init__(self, options, properties, storage_type):
self.dbPropKeys = DbPropKeys(
JDBC_DATABASE_PROPERTY,
JDBC_DRIVER_PROPERTY,
JDBC_HOSTNAME_PROPERTY,
JDBC_PORT_PROPERTY,
- JDBC_SCHEMA_PROPERTY,
+ JDBC_DATABASE_NAME_PROPERTY,
JDBC_URL_PROPERTY)
- self.dbAuthKeys = AuthenticationKeys(
+ self.dbAuthKeys = SqlServerAuthenticationKeys(
JDBC_USE_INTEGRATED_AUTH_PROPERTY,
JDBC_USER_NAME_PROPERTY,
JDBC_PASSWORD_PROPERTY,
@@ -362,15 +335,13 @@ class SQLServerAmbariDBConfig(SQLServerConfig):
JDBC_PASSWORD_FILENAME
)
- super(SQLServerAmbariDBConfig, self).__init__(options, properties)
+ super(SQLServerAmbariDBConfig, self).__init__(options, properties, storage_type)
if self.database_name is None or self.database_name is "":
self.database_name = AMBARI_DATABASE_NAME
self.persistence_property = PERSISTENCE_TYPE_PROPERTY
- self.db_title = "ambari"
-
self.env_var_db_name ='AMBARIDBNAME'
self.env_var_db_log_name = 'AMBARIDBLOGNAME'
self.env_var_db_owner = 'AMBARIDBOWNER'
@@ -389,13 +360,13 @@ class SQLServerAmbariDBConfig(SQLServerConfig):
def _setup_remote_server(self, properties):
super(SQLServerAmbariDBConfig, self)._setup_remote_server(properties)
- properties.process_pair(JDBC_RCA_DRIVER_PROPERTY, self.driver_name)
+ properties.process_pair(JDBC_RCA_DRIVER_PROPERTY, self.driver_class_name)
properties.process_pair(JDBC_RCA_HOSTNAME_PROPERTY, ensure_double_backslashes(self.database_host))
if self.database_port is not None and self.database_port != "":
properties.process_pair(JDBC_RCA_PORT_PROPERTY, self.database_port)
properties.process_pair(JDBC_RCA_SCHEMA_PROPERTY, self.database_name)
- authKeys = AuthenticationKeys(
+ authKeys = SqlServerAuthenticationKeys(
JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY,
JDBC_RCA_USER_NAME_PROPERTY,
JDBC_RCA_PASSWORD_FILE_PROPERTY,
@@ -407,10 +378,11 @@ class SQLServerAmbariDBConfig(SQLServerConfig):
properties.process_pair(JDBC_RCA_URL_PROPERTY, self.database_url)
pass
-
+#
# SQL Server Metrics database configuration and setup
+#
class SQLServerMetricsDBConfig(SQLServerConfig):
- def __init__(self, options, properties):
+ def __init__(self, options, properties, storage_type):
self.dbPropKeys = DbPropKeys(
JDBC_METRICS_DATABASE_PROPERTY,
JDBC_METRICS_DRIVER_PROPERTY,
@@ -418,7 +390,7 @@ class SQLServerMetricsDBConfig(SQLServerConfig):
JDBC_METRICS_PORT_PROPERTY,
JDBC_METRICS_SCHEMA_PROPERTY,
JDBC_METRICS_URL_PROPERTY)
- self.dbAuthKeys = AuthenticationKeys(
+ self.dbAuthKeys = SqlServerAuthenticationKeys(
JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY,
JDBC_METRICS_USER_NAME_PROPERTY,
JDBC_METRICS_PASSWORD_PROPERTY,
@@ -426,7 +398,7 @@ class SQLServerMetricsDBConfig(SQLServerConfig):
JDBC_METRICS_PASSWORD_FILENAME
)
- super(SQLServerMetricsDBConfig, self).__init__(options, properties)
+ super(SQLServerMetricsDBConfig, self).__init__(options, properties, storage_type)
self.database_name = METRICS_DATABASE_NAME
@@ -448,14 +420,10 @@ class SQLServerMetricsDBConfig(SQLServerConfig):
self.drop_tables_script_file = "resources" + os.sep + "Hadoop-Metrics-SQLServer-DROP.sql"
pass
-
-# SQL Server database
-class SQLServerDatabase:
- def __init__(self):
- #Init the database connection here
- pass
-
- def get_running_status(self):
- #if the connection is active, return running
- #else return stopped
- return DB_STATUS_RUNNING_DEFAULT
+def createSQLServerConfig(options, properties, storage_type, dbId):
+ if dbId == "Ambari":
+ return SQLServerAmbariDBConfig(options, properties, storage_type)
+ elif dbId == "Metrics":
+ return SQLServerMetricsDBConfig(options, properties, storage_type)
+ else:
+ raise FatalException(-1, "Invalid database requested: " + str(dbId))
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/properties.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/properties.py b/ambari-server/src/main/python/ambari_server/properties.py
index 8e00762..ffdfe8d 100644
--- a/ambari-server/src/main/python/ambari_server/properties.py
+++ b/ambari-server/src/main/python/ambari_server/properties.py
@@ -98,6 +98,9 @@ class Properties(object):
self.process_pair(key, value)
def process_pair(self, key, value):
+ """
+ Adds or overrides the property with the given key.
+ """
oldkey = key
oldvalue = value
keyparts = self.bspacere.split(key)
@@ -185,7 +188,9 @@ class Properties(object):
def store(self, out, header=""):
""" Write the properties list to the stream 'out' along
- with the optional 'header' """
+ with the optional 'header'
+ This function will attempt to close the file handler once it's done.
+ """
if out.mode[0] != 'w':
raise ValueError, 'Steam should be opened in write mode!'
try:
@@ -198,9 +203,11 @@ class Properties(object):
for prop, val in self._origprops.items():
if val is not None:
out.write(''.join((prop, '=', val, '\n')))
- out.close()
except IOError:
raise
+ finally:
+ if out:
+ out.close()
def store_ordered(self, out, header=""):
""" Write the properties list to the stream 'out' along
@@ -218,6 +225,8 @@ class Properties(object):
val = self._origprops[key]
if val is not None:
out.write(''.join((key, '=', val, '\n')))
- out.close()
except IOError:
raise
+ finally:
+ if out:
+ out.close()
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py b/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py
index 9d138a1..3589972 100644
--- a/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py
+++ b/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py
@@ -92,12 +92,12 @@ class ResourceFilesKeeper():
custom_actions_root = os.path.join(self.resources_dir,self.CUSTOM_ACTIONS_DIR)
self.dbg_out("Updating archive for {0} dir at {1}...".format(self.CUSTOM_ACTIONS_DIR,
custom_actions_root))
-
+
# agent host scripts
host_scripts_root = os.path.join(self.resources_dir,self.HOST_SCRIPTS_DIR)
self.dbg_out("Updating archive for {0} dir at {1}...".format(self.HOST_SCRIPTS_DIR,
host_scripts_root))
-
+
# update the directories so that the .hash is generated
self.update_directory_archive(custom_actions_root)
self.update_directory_archive(host_scripts_root)
[3/7] ambari git commit: AMBARI-8373 Refactor the OS-dependent Ambari
Server Windows components
Posted by fb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/serverSetup_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverSetup_linux.py b/ambari-server/src/main/python/ambari_server/serverSetup_linux.py
deleted file mode 100644
index b5436e0..0000000
--- a/ambari-server/src/main/python/ambari_server/serverSetup_linux.py
+++ /dev/null
@@ -1,795 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-import glob
-import optparse
-
-import os
-import re
-import socket
-
-from ambari_commons.exceptions import *
-from ambari_commons.logging_utils import *
-from ambari_commons.os_linux import run_os_command
-from ambari_server.dbConfiguration_linux import SERVICE_CMD, PG_HBA_CONF_FILE_BACKUP
-from ambari_server.serverConfiguration import *
-from ambari_server.serverConfiguration_linux import JAVA_SHARE_PATH
-from ambari_server.setupSecurity import *
-from ambari_server.userInput import get_YN_input, get_validated_string_input
-from ambari_server import utils
-
-# selinux commands
-GET_SE_LINUX_ST_CMD = utils.locate_file('sestatus', '/usr/sbin')
-SE_SETENFORCE_CMD = "setenforce 0"
-SE_STATUS_DISABLED = "disabled"
-SE_STATUS_ENABLED = "enabled"
-SE_MODE_ENFORCING = "enforcing"
-SE_MODE_PERMISSIVE = "permissive"
-
-# Non-root user setup commands
-NR_USER_COMMENT = "Ambari user"
-NR_GET_OWNER_CMD = 'stat -c "%U" {0}'
-NR_USERADD_CMD = 'useradd -M --comment "{1}" ' \
- '--shell %s -d /var/lib/ambari-server/keys/ {0}' % utils.locate_file('nologin', '/sbin')
-NR_SET_USER_COMMENT_CMD = 'usermod -c "{0}" {1}'
-
-NR_USER_CHANGE_PROMPT = "Ambari-server daemon is configured to run under user '{0}'. Change this setting [y/n] (n)? "
-NR_USER_CUSTOMIZE_PROMPT = "Customize user account for ambari-server daemon [y/n] (n)? "
-NR_DEFAULT_USER = "root"
-
-# jdk commands
-JDK_URL_PROPERTIES = ["jdk1.7.url", "jdk1.6.url"]
-JCE_URL_PROPERTIES = ["jce_policy1.7.url", "jce_policy1.6.url"]
-JDK_VERSION_REs = ["(jdk.*)/jre", "Creating (jdk.*)/jre"]
-JDK_CHOICE_PROMPT = "[1] - Oracle JDK 1.7\n[2] - Oracle JDK 1.6\n[3] - Custom JDK\n==============================================================================\nEnter choice ({0}): "
-JDK_VALID_CHOICES = "^[123]$"
-CUSTOM_JDK_NUMBER = "3"
-JDK_MIN_FILESIZE = 5000
-CREATE_JDK_DIR_CMD = "/bin/mkdir -p {0}"
-MAKE_FILE_EXECUTABLE_CMD = "chmod a+x {0}"
-
-JDK_DOWNLOAD_CMD = "curl --create-dirs -o {0} {1}"
-JDK_DOWNLOAD_SIZE_CMD = "curl -I {0}"
-
-# use --no-same-owner when running as root to prevent uucp as the user (AMBARI-6478)
-UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}"
-
-
-#JDBC
-DATABASE_INDEX = 0
-USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
-DATABASE_NAMES = ["postgres", "oracle", "mysql"]
-DATABASE_STORAGE_NAMES = ["Database", "Service", "Database"]
-DATABASE_PORTS = ["5432", "1521", "3306"]
-DATABASE_DRIVER_NAMES = ["org.postgresql.Driver", "oracle.jdbc.driver.OracleDriver", "com.mysql.jdbc.Driver"]
-DATABASE_CONNECTION_STRINGS = [
- "jdbc:postgresql://{0}:{1}/{2}",
- "jdbc:oracle:thin:@{0}:{1}/{2}",
- "jdbc:mysql://{0}:{1}/{2}"]
-DATABASE_CONNECTION_STRINGS_ALT = [
- "jdbc:postgresql://{0}:{1}/{2}",
- "jdbc:oracle:thin:@{0}:{1}:{2}",
- "jdbc:mysql://{0}:{1}/{2}"]
-ORACLE_SID_PATTERN = "jdbc:oracle:thin:@.+:.+/.+"
-ORACLE_SNAME_PATTERN = "jdbc:oracle:thin:@.+:.+:.+"
-
-DATABASE_CLI_TOOLS = [["psql"], ["sqlplus", "sqlplus64"], ["mysql"]]
-DATABASE_CLI_TOOLS_DESC = ["psql", "sqlplus", "mysql"]
-DATABASE_CLI_TOOLS_USAGE = ['su -postgres --command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'"',
- 'sqlplus {1}/{2} < {0} ',
- 'mysql --user={1} --password={2} {3}<{0}']
-
-MYSQL_INIT_SCRIPT = '/var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql'
-DATABASE_INIT_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-CREATE.sql',
- '/var/lib/ambari-server/resources/Ambari-DDL-Oracle-CREATE.sql',
- MYSQL_INIT_SCRIPT]
-DATABASE_DROP_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-DROP.sql',
- '/var/lib/ambari-server/resources/Ambari-DDL-Oracle-DROP.sql',
- '/var/lib/ambari-server/resources/Ambari-DDL-MySQL-DROP.sql']
-
-JDBC_PROPERTIES_PREFIX = "server.jdbc.properties."
-DATABASE_JDBC_PROPERTIES = [
- [ ],
- [
- ["oracle.net.CONNECT_TIMEOUT", "2000"], # socket level timeout
- ["oracle.net.READ_TIMEOUT", "2000"], # socket level timeout
- ["oracle.jdbc.ReadTimeout", "8000"] # query fetch timeout
- ],
- [ ]
- ]
-
-POSTGRES_EXEC_ARGS = "-h {0} -p {1} -d {2} -U {3} -f {4} -v username='{3}'"
-ORACLE_EXEC_ARGS = "-S -L '{0}/{1}@(description=(address=(protocol=TCP)(host={2})(port={3}))(connect_data=({6}={4})))' @{5} {0}"
-MYSQL_EXEC_ARGS_WITH_USER_VARS = "--host={0} --port={1} --user={2} --password={3} {4} " \
- "-e\"set @schema=\'{4}\'; set @username=\'{2}\'; source {5};\""
-MYSQL_EXEC_ARGS_WO_USER_VARS = "--force --host={0} --port={1} --user={2} --password={3} --database={4} < {5} 2> /dev/null"
-MYSQL_UPGRADE_STACK_ARGS = "--host={0} --port={1} --user={2} --password={3} --database={4} " \
- "-e\"set @stackName=\'{6}\'; set @stackVersion=\'{7}\'; source {5};\""
-
-ORACLE_UPGRADE_STACK_ARGS = "-S -L '{0}/{1}@(description=(address=(protocol=TCP)(host={2})(port={3}))(connect_data=({6}={4})))' @{5} {7} {8}"
-
-JDBC_PATTERNS = {"oracle": "*ojdbc*.jar", "mysql": "*mysql*.jar"}
-DATABASE_FULL_NAMES = {"oracle": "Oracle", "mysql": "MySQL", "postgres": "PostgreSQL"}
-JDBC_DB_OPTION_VALUES = ["postgres", "mysql", "oracle"]
-JDBC_DB_DEFAULT_DRIVER = {"postgresql" : "postgresql-jdbc.jar", "mysql" : "mysql-connector-java.jar", "oracle" : "ojdbc6.jar"}
-ORACLE_DB_ID_TYPES = ["Service Name", "SID"]
-
-
-DEFAULT_DB_NAME = "ambari"
-
-
-MESSAGE_ERROR_NOT_ROOT = 'Ambari-server setup should be run with root-level privileges'
-
-MESSAGE_CHECK_FIREWALL = 'Checking iptables...'
-
-class FirewallChecks(object):
- def __init__(self):
-
- self.FIREWALL_SERVICE_NAME = "iptables"
- self.SERVICE_CMD = SERVICE_CMD
- self.SERVICE_SUBCMD = "status"
-
- def get_command(self):
- return "%s %s %s" % (self.SERVICE_CMD, self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD)
-
- def check_result(self, retcode, out, err):
- return retcode == 0
-
- def check_iptables(self):
- retcode, out, err = run_os_command(self.get_command())
- if err and len(err) > 0:
- print err
- if self.check_result(retcode, out, err):
- print_warning_msg("%s is running. Confirm the necessary Ambari ports are accessible. " %
- self.FIREWALL_SERVICE_NAME +
- "Refer to the Ambari documentation for more details on ports.")
- ok = get_YN_input("OK to continue [y/n] (y)? ", True)
- if not ok:
- raise FatalException(1, None)
-
- def get_running_result(self):
- # To support test code. Expected ouput from run_os_command.
- return (0, "", "")
-
- def get_stopped_result(self):
- # To support test code. Expected output from run_os_command.
- return (3, "", "")
-
-
-class UbuntuFirewallChecks(FirewallChecks):
- def __init__(self):
- super(UbuntuFirewallChecks, self).__init__()
-
- self.FIREWALL_SERVICE_NAME = "ufw"
- self.SERVICE_CMD = utils.locate_file('service', '/usr/sbin')
-
- def check_result(self, retcode, out, err):
- # On ubuntu, the status command returns 0 whether running or not
- return out and len(out) > 0 and out.strip() != "ufw stop/waiting"
-
- def get_running_result(self):
- # To support test code. Expected ouput from run_os_command.
- return (0, "ufw start/running", "")
-
- def get_stopped_result(self):
- # To support test code. Expected output from run_os_command.
- return (0, "ufw stop/waiting", "")
-
-
-class Fedora18FirewallChecks(FirewallChecks):
- def __init__(self):
- self.FIREWALL_SERVICE_NAME = "firewalld.service"
-
- def get_command(self):
- return "systemctl is-active firewalld.service"
-
-
-class OpenSuseFirewallChecks(FirewallChecks):
- def __init__(self):
- self.FIREWALL_SERVICE_NAME = "SuSEfirewall2"
-
- def get_command(self):
- return "/sbin/SuSEfirewall2 status"
-
-
-def get_firewall_object():
- if OS_TYPE == OSConst.OS_UBUNTU:
- return UbuntuFirewallChecks()
- elif OS_TYPE == OSConst.OS_FEDORA and int(OS_VERSION) >= 18:
- return Fedora18FirewallChecks()
- elif OS_TYPE == OSConst.OS_OPENSUSE:
- return OpenSuseFirewallChecks()
- else:
- return FirewallChecks()
-
-
-def get_firewall_object_types():
- # To support test code, so tests can loop through the types
- return (FirewallChecks,
- UbuntuFirewallChecks,
- Fedora18FirewallChecks,
- OpenSuseFirewallChecks)
-
-
-def os_check_firewall():
- return get_firewall_object().check_iptables()
-
-
-#
-# Checks SELinux
-#
-def check_selinux():
- try:
- retcode, out, err = run_os_command(GET_SE_LINUX_ST_CMD)
- se_status = re.search('(disabled|enabled)', out).group(0)
- print "SELinux status is '" + se_status + "'"
- if se_status == SE_STATUS_DISABLED:
- return 0
- else:
- try:
- se_mode = re.search('(enforcing|permissive)', out).group(0)
- except AttributeError:
- err = "Error determining SELinux mode. Exiting."
- raise FatalException(1, err)
- print "SELinux mode is '" + se_mode + "'"
- if se_mode == SE_MODE_ENFORCING:
- print "Temporarily disabling SELinux"
- run_os_command(SE_SETENFORCE_CMD)
- print_warning_msg(
- "SELinux is set to 'permissive' mode and temporarily disabled.")
- ok = get_YN_input("OK to continue [y/n] (y)? ", True)
- if not ok:
- raise FatalException(1, None)
- return 0
- except OSError:
- print_warning_msg("Could not run {0}: OK".format(GET_SE_LINUX_ST_CMD))
- return 0
-
-def disable_security_enhancements():
- print 'Checking SELinux...'
- err = ''
- retcode = check_selinux()
- if not retcode == 0:
- err = 'Failed to disable SELinux. Exiting.'
- return (retcode, err)
-
-
-#
-# User account creation
-#
-
-def os_create_custom_user():
- user = get_validated_string_input(
- "Enter user account for ambari-server daemon (root):",
- "root",
- "^[a-z_][a-z0-9_-]{1,31}$",
- "Invalid username.",
- False
- )
-
- print_info_msg("Trying to create user {0}".format(user))
- command = NR_USERADD_CMD.format(user, NR_USER_COMMENT)
- retcode, out, err = run_os_command(command)
- if retcode == 9: # 9 = username already in use
- print_info_msg("User {0} already exists, "
- "skipping user creation".format(user))
-
- elif retcode != 0: # fail
- print_warning_msg("Can't create user {0}. Command {1} "
- "finished with {2}: \n{3}".format(user, command, retcode, err))
- return retcode, None
-
- print_info_msg("User configuration is done.")
- return 0, user
-
-
-#
-# JDK Setup
-#
-
-def os_install_jdk(java_inst_file, java_home_dir):
- print "Installing JDK to {0}".format(java_home_dir)
- retcode, out, err = run_os_command(CREATE_JDK_DIR_CMD.format(java_home_dir))
- savedPath = os.getcwd()
- os.chdir(java_home_dir)
-
- if java_inst_file.endswith(".bin"):
- retcode, out, err = run_os_command(MAKE_FILE_EXECUTABLE_CMD.format(java_inst_file))
- retcode, out, err = run_os_command(java_inst_file + ' -noregister')
- elif java_inst_file.endswith(".gz"):
- retcode, out, err = run_os_command(UNTAR_JDK_ARCHIVE.format(java_inst_file))
- else:
- err = "JDK installation failed.Unknown file mask."
- raise FatalException(1, err)
-
- os.chdir(savedPath)
-
- if retcode != 0:
- err = "Installation of JDK returned exit code %s" % retcode
- raise FatalException(retcode, err)
-
- print "Successfully installed JDK to {0}".format(java_home_dir)
- return (retcode, out)
-
-def os_ensure_java_home_env_var_is_set(java_home_var):
- if not os.environ.has_key(JAVA_HOME) or os.environ[JAVA_HOME] != java_home_var:
- os.system("SETX {0} {1} /M".format(JAVA_HOME, java_home_var))
- os.environ[JAVA_HOME] = java_home_var
- pass
-
-
-#
-# JDBC Setup
-#
-
-def os_check_jdbc_options(options):
- return (options.jdbc_driver is not None and options.jdbc_db is not None)
-
-#Check if required jdbc drivers present
-def os_find_jdbc_driver(args):
- if args.dbms in JDBC_PATTERNS.keys():
- drivers = []
- drivers.extend(glob.glob(JAVA_SHARE_PATH + os.sep + JDBC_PATTERNS[args.dbms]))
- if drivers:
- return drivers
- return -1
- return 0
-
-def os_setup_jdbc_drivers(args):
- result = os_find_jdbc_driver(args)
-
- msg = 'Before starting Ambari Server, ' \
- 'you must copy the {0} JDBC driver JAR file to {1}.'.format(
- DATABASE_FULL_NAMES[args.dbms],
- JAVA_SHARE_PATH)
-
- if result == -1:
- if SILENT:
- print_error_msg(msg)
- raise FatalException(-1, msg)
- else:
- print_warning_msg(msg)
- raw_input(PRESS_ENTER_MSG)
- result = os_find_jdbc_driver(args)
- if result == -1:
- print_error_msg(msg)
- raise FatalException(-1, msg)
-
- # Check if selected RDBMS requires drivers to copy
- if type(result) is not int:
- print 'Copying JDBC drivers to server resources...'
- try:
- resources_dir = args[RESOURCES_DIR_PROPERTY]
- except KeyError:
- print_error_msg("There is no value for " + RESOURCES_DIR_PROPERTY + "in " + AMBARI_PROPERTIES_FILE)
- return -1
-
- db_name = DATABASE_FULL_NAMES[args.dbms].lower()
- jdbc_symlink = os.path.join(resources_dir, db_name + "-jdbc-driver.jar")
- db_default_driver_path = os.path.join(JAVA_SHARE_PATH, JDBC_DB_DEFAULT_DRIVER[db_name])
-
- if os.path.lexists(jdbc_symlink):
- os.remove(jdbc_symlink)
-
- copy_status = copy_files(result, resources_dir)
-
- if not copy_status == 0:
- raise FatalException(-1, "Failed to copy JDBC drivers to server resources")
-
- if db_default_driver_path in result:
- os.symlink(os.path.join(resources_dir, JDBC_DB_DEFAULT_DRIVER[db_name]), jdbc_symlink)
-
-def os_load_default_db_properties(args):
- args.persistence_type = 'local'
- args.dbms = DATABASE_NAMES[DATABASE_INDEX]
- args.database_host = "localhost"
- args.database_port = DATABASE_PORTS[DATABASE_INDEX]
- args.database_name = DEFAULT_DB_NAME
- args.database_username = "ambari"
- args.database_password = "bigdata"
- args.sid_or_sname = "sname"
- pass
-
-# Check if jdbc user is changed
-def is_jdbc_user_changed(args):
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return None
-
- previos_user = properties[JDBC_USER_NAME_PROPERTY]
- new_user = args.database_username
-
- if previos_user and new_user:
- if previos_user != new_user:
- return True
- else:
- return False
-
- return None
-
-def os_setup_database(options):
- if is_local_database(options):
- os_setup_local_database(options)
- else:
- os_setup_remote_database(options)
-
-def os_setup_local_database(options):
- #check if jdbc user is changed
- is_user_changed = is_jdbc_user_changed(options)
-
- print 'Default properties detected. Using built-in database.'
- os_store_local_properties(options)
-
- print 'Checking PostgreSQL...'
- pg_status, retcode, out, err = check_postgre_up()
- if not retcode == 0:
- err = 'Unable to start PostgreSQL server. Status {0}. {1}.' \
- ' Exiting'.format(pg_status, err)
- raise FatalException(retcode, err)
-
- print 'Configuring local database...'
- retcode, outdata, errdata = setup_db(options)
- if not retcode == 0:
- err = 'Running database init script was failed. {0}. Exiting.'.format(errdata)
- raise FatalException(retcode, err)
-
- if is_user_changed:
- #remove backup for pg_hba in order to reconfigure postgres
- remove_file(PG_HBA_CONF_FILE_BACKUP)
-
- print 'Configuring PostgreSQL...'
- retcode, out, err = configure_postgres()
- if not retcode == 0:
- err = 'Unable to configure PostgreSQL server. {0} Exiting'.format(err)
- raise FatalException(retcode, err)
- pass
-
-def os_setup_remote_database(options):
- retcode = os_store_remote_properties(options)
- if retcode != 0:
- err = 'Unable to save config file'
- raise FatalException(retcode, err)
-
- os_setup_jdbc_drivers(options)
-
- print 'Configuring remote database connection properties...'
- retcode = setup_remote_db(options)
- if retcode == -1:
- err = "Remote database setup aborted."
- raise NonFatalException(err)
-
- if not retcode == 0:
- err = 'Error while configuring connection properties. Exiting'
- raise FatalException(retcode, err)
- pass
-
-def os_reset_database(options):
- pass
-
-
-### Postgres ###
-
-
-def configure_pg_hba_ambaridb_users():
- args = optparse.Values()
- configure_database_username_password(args)
-
- with open(PG_HBA_CONF_FILE, "a") as pgHbaConf:
- pgHbaConf.write("\n")
- pgHbaConf.write("local all " + args.database_username +
- ",mapred md5")
- pgHbaConf.write("\n")
- pgHbaConf.write("host all " + args.database_username +
- ",mapred 0.0.0.0/0 md5")
- pgHbaConf.write("\n")
- pgHbaConf.write("host all " + args.database_username +
- ",mapred ::/0 md5")
- pgHbaConf.write("\n")
- retcode, out, err = run_os_command(PG_HBA_RELOAD_CMD)
- if not retcode == 0:
- raise FatalException(retcode, err)
-
-
-def configure_pg_hba_postgres_user():
- postgresString = "all postgres"
- for line in fileinput.input(PG_HBA_CONF_FILE, inplace=1):
- print re.sub('all\s*all', postgresString, line),
- os.chmod(PG_HBA_CONF_FILE, 0644)
-
-
-def configure_postgresql_conf():
- listenAddress = "listen_addresses = '*' #"
- for line in fileinput.input(POSTGRESQL_CONF_FILE, inplace=1):
- print re.sub('#+listen_addresses.*?(#|$)', listenAddress, line),
- os.chmod(POSTGRESQL_CONF_FILE, 0644)
-
-
-def configure_postgres():
- if os.path.isfile(PG_HBA_CONF_FILE):
- if not os.path.isfile(PG_HBA_CONF_FILE_BACKUP):
- shutil.copyfile(PG_HBA_CONF_FILE, PG_HBA_CONF_FILE_BACKUP)
- else:
- #Postgres has been configured before, must not override backup
- print "Backup for pg_hba found, reconfiguration not required"
- return 0, "", ""
- configure_pg_hba_postgres_user()
- configure_pg_hba_ambaridb_users()
- os.chmod(PG_HBA_CONF_FILE, 0644)
- configure_postgresql_conf()
- #restart postgresql if already running
- pg_status, retcode, out, err = get_postgre_status()
- if pg_status == PG_STATUS_RUNNING:
- retcode, out, err = restart_postgres()
- return retcode, out, err
- return 0, "", ""
-
-
-def restart_postgres():
- print "Restarting PostgreSQL"
- process = subprocess.Popen(PG_RESTART_CMD.split(' '),
- stdout=subprocess.PIPE,
- stdin=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
- time.sleep(5)
- result = process.poll()
- if result is None:
- print_info_msg("Killing restart PostgresSQL process")
- process.kill()
- pg_status, retcode, out, err = get_postgre_status()
- # SUSE linux set status of stopped postgresql proc to unused
- if pg_status == "unused" or pg_status == "stopped":
- print_info_msg("PostgreSQL is stopped. Restarting ...")
- retcode, out, err = run_os_command(PG_START_CMD)
- return retcode, out, err
- return 0, "", ""
-
-
-# todo: check if the scheme is already exist
-
-
-def setup_db(args):
- #password access to ambari-server and mapred
- configure_database_username_password(args)
- dbname = args.database_name
- scriptFile = args.init_script_file
- username = args.database_username
- password = args.database_password
-
- #setup DB
- command = SETUP_DB_CMD[:]
- command[-1] = command[-1].format(scriptFile, username, password, dbname)
-
- for i in range(SETUP_DB_CONNECT_ATTEMPTS):
- sys.stdout.write('Connecting to local database...')
- retcode, outdata, errdata = run_os_command(command)
- if retcode == 0:
- print 'done.'
- return retcode, outdata, errdata
- timeOutMsg = 'connection timed out'
- if (i+1) < SETUP_DB_CONNECT_ATTEMPTS:
- timeOutMsg += '...retrying (%d)' % (i+1)
- print timeOutMsg
- time.sleep(SETUP_DB_CONNECT_TIMEOUT)
-
- print 'unable to connect to database'
- print_error_msg(errdata)
- return retcode, outdata, errdata
-
-
-def execute_db_script(args, file):
- #password access to ambari-server and mapred
- configure_database_username_password(args)
- dbname = args.database_name
- username = args.database_username
- password = args.database_password
- command = SETUP_DB_CMD[:]
- command[-1] = command[-1].format(file, username, password, dbname)
- retcode, outdata, errdata = run_os_command(command)
- if not retcode == 0:
- print errdata
- return retcode
-
-
-def check_db_consistency(args, file):
- #password access to ambari-server and mapred
- configure_database_username_password(args)
- dbname = args.database_name
- username = args.database_username
- password = args.database_password
- command = SETUP_DB_CMD[:]
- command[-1] = command[-1].format(file, username, password, dbname)
- retcode, outdata, errdata = run_os_command(command)
- if not retcode == 0:
- print errdata
- return retcode
- else:
- # Assumes that the output is of the form ...\n<count>
- print_info_msg("Parsing output: " + outdata)
- lines = outdata.splitlines()
- if (lines[-1] == '3' or lines[-1] == '0'):
- return 0
- return -1
-
-
-def get_postgre_status():
- retcode, out, err = run_os_command(PG_ST_CMD)
- try:
- pg_status = re.search('(stopped|running)', out, re.IGNORECASE).group(0).lower()
- except AttributeError:
- pg_status = None
- return pg_status, retcode, out, err
-
-
-def check_postgre_up():
- pg_status, retcode, out, err = get_postgre_status()
- if pg_status == PG_STATUS_RUNNING:
- print_info_msg("PostgreSQL is running")
- return pg_status, 0, out, err
- else:
- # run initdb only on non ubuntu systems as ubuntu does not have initdb cmd.
- if OS_TYPE != OSConst.OS_UBUNTU:
- print "Running initdb: This may take upto a minute."
- retcode, out, err = run_os_command(PG_INITDB_CMD)
- if retcode == 0:
- print out
- print "About to start PostgreSQL"
- try:
- process = subprocess.Popen(PG_START_CMD.split(' '),
- stdout=subprocess.PIPE,
- stdin=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
- if OS_TYPE == OSConst.OS_SUSE:
- time.sleep(20)
- result = process.poll()
- print_info_msg("Result of postgres start cmd: " + str(result))
- if result is None:
- process.kill()
- pg_status, retcode, out, err = get_postgre_status()
- else:
- retcode = result
- else:
- out, err = process.communicate()
- retcode = process.returncode
- if pg_status == PG_STATUS_RUNNING:
- print_info_msg("Postgres process is running. Returning...")
- return pg_status, 0, out, err
- except (Exception), e:
- pg_status, retcode, out, err = get_postgre_status()
- if pg_status == PG_STATUS_RUNNING:
- return pg_status, 0, out, err
- else:
- print_error_msg("Postgres start failed. " + str(e))
- return pg_status, retcode, out, err
-
-def get_validated_db_name(database_name):
- return get_validated_string_input(
- DATABASE_STORAGE_NAMES[DATABASE_INDEX] + " Name ("
- + database_name + "): ",
- database_name,
- ".*",
- "Invalid " + DATABASE_STORAGE_NAMES[DATABASE_INDEX] + " name.",
- False
- )
-
-def get_validated_service_name(service_name, index):
- return get_validated_string_input(
- ORACLE_DB_ID_TYPES[index] + " (" + service_name + "): ",
- service_name,
- ".*",
- "Invalid " + ORACLE_DB_ID_TYPES[index] + ".",
- False
- )
-
-def get_pass_file_path(conf_file):
- return os.path.join(os.path.dirname(conf_file),
- JDBC_PASSWORD_FILENAME)
-
-# Store local database connection properties
-def os_store_local_properties(args):
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return -1
-
- isSecure = get_is_secure(properties)
-
- properties.removeOldProp(JDBC_SCHEMA_PROPERTY)
- properties.removeOldProp(JDBC_HOSTNAME_PROPERTY)
- properties.removeOldProp(JDBC_RCA_DRIVER_PROPERTY)
- properties.removeOldProp(JDBC_RCA_URL_PROPERTY)
- properties.removeOldProp(JDBC_PORT_PROPERTY)
- properties.removeOldProp(JDBC_DRIVER_PROPERTY)
- properties.removeOldProp(JDBC_URL_PROPERTY)
- properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "local")
- properties.process_pair(JDBC_DATABASE_PROPERTY, args.database_name)
- properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username)
- properties.process_pair(JDBC_PASSWORD_PROPERTY,
- store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
-
- if isSecure:
- encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password)
- if args.database_password != encrypted_password:
- properties.process_pair(JDBC_PASSWORD_PROPERTY, encrypted_password)
- pass
- pass
-
- return 0
-
-
-# Store set of properties for remote database connection
-def os_store_remote_properties(args):
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return -1
-
- isSecure = get_is_secure(properties)
-
- properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "remote")
-
- properties.process_pair(JDBC_DATABASE_PROPERTY, args.dbms)
- properties.process_pair(JDBC_HOSTNAME_PROPERTY, args.database_host)
- properties.process_pair(JDBC_PORT_PROPERTY, args.database_port)
- properties.process_pair(JDBC_SCHEMA_PROPERTY, args.database_name)
-
- properties.process_pair(JDBC_DRIVER_PROPERTY, DATABASE_DRIVER_NAMES[DATABASE_INDEX])
- # fully qualify the hostname to make sure all the other hosts can connect
- # to the jdbc hostname since its passed onto the agents for RCA
- jdbc_hostname = args.database_host
- if (args.database_host == "localhost"):
- jdbc_hostname = socket.getfqdn()
-
- connectionStringFormat = DATABASE_CONNECTION_STRINGS
- if args.sid_or_sname == "sid":
- connectionStringFormat = DATABASE_CONNECTION_STRINGS_ALT
- properties.process_pair(JDBC_URL_PROPERTY,
- connectionStringFormat[DATABASE_INDEX].format(jdbc_hostname, args.database_port,
- args.database_name))
- properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username)
- properties.process_pair(JDBC_PASSWORD_PROPERTY,
- store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
-
- # save any other defined properties to pass to JDBC
- if DATABASE_INDEX < len(DATABASE_JDBC_PROPERTIES):
- for pair in DATABASE_JDBC_PROPERTIES[DATABASE_INDEX]:
- properties.process_pair(JDBC_PROPERTIES_PREFIX + pair[0], pair[1])
-
- if isSecure:
- encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password)
- if encrypted_password != args.database_password:
- properties.process_pair(JDBC_PASSWORD_PROPERTY, encrypted_password)
- pass
-
- properties.process_pair(JDBC_RCA_DRIVER_PROPERTY, DATABASE_DRIVER_NAMES[DATABASE_INDEX])
- properties.process_pair(JDBC_RCA_URL_PROPERTY,
- connectionStringFormat[DATABASE_INDEX].format(jdbc_hostname, args.database_port,
- args.database_name))
- properties.process_pair(JDBC_RCA_USER_NAME_PROPERTY, args.database_username)
- properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY,
- store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
- if isSecure:
- encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password)
- if encrypted_password != args.database_password:
- properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY, encrypted_password)
- pass
-
- return 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/serverSetup_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverSetup_windows.py b/ambari-server/src/main/python/ambari_server/serverSetup_windows.py
deleted file mode 100644
index a906ef5..0000000
--- a/ambari-server/src/main/python/ambari_server/serverSetup_windows.py
+++ /dev/null
@@ -1,313 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import optparse
-import os
-import socket
-import string
-
-from _winreg import (OpenKey, EnumValue, HKEY_LOCAL_MACHINE, KEY_READ, CloseKey, KEY_WRITE, QueryValueEx, SetValueEx,
- REG_EXPAND_SZ)
-
-from ambari_commons import os_utils
-
-from ambari_commons.exceptions import *
-from ambari_commons.logging_utils import *
-from ambari_commons.os_windows import run_os_command, UserHelper
-from ambari_server.dbConfiguration import DBMSConfig
-from ambari_server.serverConfiguration import *
-from ambari_server.serverConfiguration_windows import OUT_DIR
-from ambari_server.userInput import get_validated_string_input
-
-# Non-root user setup commands
-NR_USER_COMMENT = "Ambari user"
-NR_GET_OWNER_CMD = 'stat -c "%U" {0}'
-NR_USERADD_CMD = 'cmd /C net user {0} {1} /ADD'
-NR_SET_USER_COMMENT_CMD = 'usermod -c "{0}" {1}'
-
-NR_USER_CHANGE_PROMPT = "Ambari-server service is configured to run under user '{0}'. Change this setting [y/n] (n)? "
-NR_USER_CUSTOMIZE_PROMPT = "Customize user account for ambari-server service [y/n] (n)? "
-NR_DEFAULT_USER = "NT AUTHORITY\SYSTEM"
-
-SERVICE_USERNAME_KEY = "TMP_AMBARI_USERNAME"
-SERVICE_PASSWORD_KEY = "TMP_AMBARI_PASSWORD"
-
-# JDK setup choices
-JDK_DEFAULT_CONFIGS = [
- JDKRelease("jdk7.67", "Oracle JDK 7.67",
- "http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe", "jdk-7u67-windows-x64.exe",
- "http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip", "UnlimitedJCEPolicyJDK7.zip",
- None)
-]
-
-JDK_VERSION_REs = ["(jdk.*)/jre", "Creating (jdk.*)/jre"]
-JDK_PROMPT = "[{0}] {1}\n"
-JDK_CUSTOM_CHOICE_PROMPT = "[{0}] - Custom JDK\n==============================================================================\nEnter choice ({1}): "
-JDK_VALID_CHOICES = "^[{0}{1:d}]$"
-CUSTOM_JDK_NUMBER = "4"
-JDK_MIN_FILESIZE = 5000
-MAKE_FILE_EXECUTABLE_CMD = "chmod a+x {0}"
-
-JDK_DOWNLOAD_CMD = "curl --create-dirs -o {0} {1}"
-JDK_DOWNLOAD_SIZE_CMD = "curl -I {0}"
-
-# use --no-same-owner when running as root to prevent uucp as the user (AMBARI-6478)
-UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}"
-
-
-#JDBC
-USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
-DATABASE_DBMS = "sqlserver"
-DATABASE_NAME = "ambari"
-DATABASE_SERVER = "localhost\\\\SQLEXPRESS"
-DATABASE_DRIVER_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
-
-METRICS_DATABASE_NAME = "HadoopMetrics"
-
-JDBC_PATTERNS = {"sqlserver": "sqljdbc*.jar"}
-DATABASE_FULL_NAMES = {"sqlserver": "SQL Server"}
-JDBC_DB_OPTION_VALUES = ["sqlserver"]
-JDBC_DB_DEFAULT_DRIVER = {"sqlserver" : "sqljdbc4.jar"}
-
-
-ERROR_NOT_ROOT = 'Ambari-server setup should be run with administrator-level privileges'
-
-MESSAGE_CHECK_FIREWALL = 'Checking firewall status...'
-
-def os_check_firewall():
- out = run_powershell_script(CHECK_FIREWALL_SCRIPT)
- if out[0] != 0:
- print_warning_msg("Unable to check firewall status:{0}".format(out[2]))
- return False
- profiles_status = [i for i in out[1].split("\n") if not i == ""]
- if "1" in profiles_status:
- enabled_profiles = []
- if profiles_status[0] == "1":
- enabled_profiles.append("DomainProfile")
- if profiles_status[1] == "1":
- enabled_profiles.append("StandardProfile")
- if profiles_status[2] == "1":
- enabled_profiles.append("PublicProfile")
- print_warning_msg("Following firewall profiles enabled:{0}. Make sure that firewall properly configured.".format(",".join(enabled_profiles)))
- return False
- return True
-
-# No security enhancements in Windows
-def disable_security_enhancements():
- retcode = 0
- err = ''
- return (retcode, err)
-
-
-#
-# User account creation
-#
-
-def os_create_custom_user():
- user = get_validated_string_input(
- "Enter user account for ambari-server service ({0}):".format(NR_DEFAULT_USER),
- NR_DEFAULT_USER, None,
- "Invalid username.",
- False
- )
- if user == NR_DEFAULT_USER:
- return 0, user
- password = get_validated_string_input("Enter password for user {0}:".format(user), "", None, "Password", True, False)
-
- uh = UserHelper()
-
- status, message = uh.create_user(user,password)
- if status == UserHelper.USER_EXISTS:
- print_info_msg("User {0} already exists, make sure that you typed correct password for user, "
- "skipping user creation".format(user))
-
- elif status == UserHelper.ACTION_FAILED: # fail
- print_warning_msg("Can't create user {0}. Failed with message {1}".format(user, message))
- return UserHelper.ACTION_FAILED, None
-
- # setting SeServiceLogonRight to user
-
- status, message = uh.add_user_privilege(user, 'SeServiceLogonRight')
- if status == UserHelper.ACTION_FAILED:
- print_warning_msg("Can't add SeServiceLogonRight to user {0}. Failed with message {1}".format(user, message))
- return UserHelper.ACTION_FAILED, None
-
- print_info_msg("User configuration is done.")
- print_warning_msg("When using non SYSTEM user make sure that your user have read\write access to log directories and "
- "all server directories. In case of integrated authentication for SQL Server make sure that your "
- "user properly configured to use ambari and metric database.")
- #storing username and password in os.environ temporary to pass them to service
- os.environ[SERVICE_USERNAME_KEY] = user
- os.environ[SERVICE_PASSWORD_KEY] = password
- return 0, user
-
-
-#
-# JDK Setup
-#
-def populate_jdk_configs(properties, jdk_num):
- if properties.has_key(JDK_RELEASES):
- jdk_names = properties[JDK_RELEASES].split(',')
- jdks = []
- for jdk_name in jdk_names:
- jdkR = JDKRelease.from_properties(properties, jdk_name)
- jdks.append(jdkR)
- else:
- jdks = JDK_DEFAULT_CONFIGS
-
- n_config = 1
- jdk_choice_prompt = ''
- jdk_choices = ''
- for jdk in jdks:
- jdk_choice_prompt += JDK_PROMPT.format(n_config, jdk.desc)
- jdk_choices_tmp = '{0}{1:d}'.format(jdk_choices, n_config)
- jdk_choices = jdk_choices_tmp
- n_config += 1
-
- jdk_choice_prompt += JDK_CUSTOM_CHOICE_PROMPT.format(n_config, jdk_num)
- jdk_valid_choices = JDK_VALID_CHOICES.format(jdk_choices, n_config)
-
- return (jdks, jdk_choice_prompt, jdk_valid_choices, n_config)
-
-
-def os_install_jdk(java_inst_file, java_home_dir):
- print "Installing JDK to {0}".format(java_home_dir)
-
- if not os.path.exists(java_home_dir):
- os.makedirs(java_home_dir)
-
- if java_inst_file.endswith(".exe"):
- (dirname, filename) = os.path.split(java_inst_file)
- installLogFilePath = os.path.join(OUT_DIR, filename + "-install.log")
- #jre7u67.exe /s INSTALLDIR=<dir> STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-server\\jre7u67.exe-install.log
- installCmd = [
- java_inst_file,
- "/s",
- "INSTALLDIR=" + java_home_dir,
- "STATIC=1",
- "WEB_JAVA=0",
- "/L",
- installLogFilePath
- ]
- retcode, out, err = run_os_command(installCmd)
- #TODO: support .msi file installations
- #msiexec.exe jre.msi /s INSTALLDIR=<dir> STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-server\\jre7u67-install.log ?
- else:
- err = "JDK installation failed.Unknown file mask."
- raise FatalException(1, err)
-
- if retcode == 1603:
- # JDK already installed
- print "JDK already installed in {0}".format(java_home_dir)
- retcode = 0
- else:
- if retcode != 0:
- err = "Installation of JDK returned exit code %s" % retcode
- raise FatalException(retcode, err)
-
- print "Successfully installed JDK to {0}".format(java_home_dir)
-
- # Don't forget to adjust the JAVA_HOME env var
-
- return (retcode, out)
-
-def os_ensure_java_home_env_var_is_set(java_home_var):
- if not os.environ.has_key(JAVA_HOME) or os.environ[JAVA_HOME] != java_home_var:
- java_home_var_val = java_home_var.replace('\\\\', '\\')
- os.system("SETX {0} {1} /M".format(JAVA_HOME, java_home_var_val))
- os.environ[JAVA_HOME] = java_home_var
- pass
-
-#
-# JDBC Setup
-#
-
-def os_check_jdbc_options(options):
- #Only SQL Server supported, no point in checking options.jdbc_db
- return (options.jdbc_driver is not None)
-
-def os_setup_jdbc_drivers(args):
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return -1
-
- #Only support SQL Server
- dbms = DBMSConfig.create(args, properties)
- if dbms.ensure_jdbc_driver_installed(args, properties):
- # Now save the properties file
- update_properties(properties)
- pass
-
-def os_setup_database(options):
- properties = get_ambari_properties()
- if properties == -1:
- raise FatalException(-1, "Error getting ambari properties")
-
- #Ensure the default database host is set
- options.default_database_host = "localhost\\SQLEXPRESS"
-
- #Only support SQL Server
- dbmsAmbari = DBMSConfig.create(options, properties, "Ambari")
- resultA = dbmsAmbari.configure_database(options, properties)
-
- #By default, use the same server for Metrics
- options.default_database_host = dbmsAmbari.database_host
-
- dbmsMetrics = DBMSConfig.create(options, properties, "Metrics")
- resultM = dbmsMetrics.configure_database(options, properties)
-
- # Now save the properties file
- if resultA or resultM:
- update_properties(properties)
-
- dbmsAmbari.setup_database()
- dbmsMetrics.setup_database()
-
-def os_reset_database(options):
- properties = get_ambari_properties()
- if properties == -1:
- raise FatalException(-1, "Error getting ambari properties")
-
- if not (properties.getPropertyDict().has_key(JDBC_URL_PROPERTY) and
- properties.getPropertyDict().has_key(JDBC_RCA_URL_PROPERTY) and
- properties.getPropertyDict().has_key(JDBC_METRICS_URL_PROPERTY)):
- raise FatalException(-1, "Ambari Server not set up yet. Nothing to reset.")
-
- empty_options = optparse.Values()
- empty_options.silent = options.silent
- empty_options.database_host = ""
- empty_options.database_port = ""
- empty_options.database_name = ""
- empty_options.database_windows_auth = False
- empty_options.database_username = ""
- empty_options.database_password = ""
- empty_options.init_db_script_file = ""
- empty_options.cleanup_db_script_file = ""
- empty_options.init_metrics_db_script_file = ""
- empty_options.cleanup_metrics_db_script_file = ""
-
- #Only support SQL Server
- dbmsAmbari = DBMSConfig.create(empty_options, properties, "Ambari")
- dbmsAmbari.reset_database()
-
- dbmsMetrics = DBMSConfig.create(empty_options, properties, "Metrics")
- dbmsMetrics.reset_database()
- pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/setupSecurity.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/setupSecurity.py b/ambari-server/src/main/python/ambari_server/setupSecurity.py
index 57a3231..fd1ee8a 100644
--- a/ambari-server/src/main/python/ambari_server/setupSecurity.py
+++ b/ambari-server/src/main/python/ambari_server/setupSecurity.py
@@ -19,16 +19,32 @@ limitations under the License.
'''
import datetime
import fileinput
+import os
import random
+import re
+import shutil
import socket
import stat
+import string
import sys
+import tempfile
+import time
import urllib2
-from ambari_commons.exceptions import *
-from serverConfiguration import *
-from setupActions import *
-from userInput import *
+from ambari_commons.exceptions import FatalException, NonFatalException
+from ambari_commons.logging_utils import print_error_msg, print_info_msg, print_warning_msg, SILENT
+from ambari_commons.os_utils import copy_file, remove_file, search_file, set_file_permissions, is_valid_filepath, \
+ is_root, run_os_command
+from ambari_server.serverConfiguration import find_properties_file, get_ambari_properties, get_value_from_properties, \
+ get_conf_dir, get_full_ambari_classpath, read_ambari_user, find_jdk, get_prompt_default, update_properties_2, \
+ configDefaults, \
+ BOOTSTRAP_DIR_PROPERTY, GET_FQDN_SERVICE_URL, \
+ JDBC_USE_INTEGRATED_AUTH_PROPERTY, JDBC_PASSWORD_PROPERTY, JDBC_PASSWORD_FILENAME, \
+ JDBC_RCA_PASSWORD_ALIAS, JDBC_RCA_PASSWORD_FILE_PROPERTY, \
+ JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY, JDBC_METRICS_PASSWORD_PROPERTY, \
+ JDBC_METRICS_PASSWORD_ALIAS, JDBC_METRICS_PASSWORD_FILENAME, BLIND_PASSWORD
+from ambari_server.setupActions import SETUP_ACTION, LDAP_SETUP_ACTION
+from ambari_server.userInput import get_validated_filepath_input, get_validated_string_input, get_YN_input
SSL_PASSWORD_FILE = "pass.txt"
@@ -40,36 +56,25 @@ EXPRT_KSTR_CMD = "openssl pkcs12 -export -in '{0}' -inkey '{1}' -certfile '{0}'
CHANGE_KEY_PWD_CND = 'openssl rsa -in {0} -des3 -out {0}.secured -passout pass:{1}'
GET_CRT_INFO_CMD = 'openssl x509 -dates -subject -in {0}'
-#keytool commands
-keytool_bin = "keytool"
-if OSCheck.is_windows_family():
- keytool_bin = "keytool.exe"
-
-KEYTOOL_IMPORT_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + keytool_bin + " -import -alias '{1}' -storetype '{2}' -file '{3}' -storepass '{4}' -noprompt"
-KEYTOOL_DELETE_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + keytool_bin + " -delete -alias '{1}' -storepass '{2}' -noprompt"
+#keytool command
+KEYTOOL_IMPORT_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + configDefaults.keytool_bin + " -import -alias '{1}' -storetype '{2}' -file '{3}' -storepass '{4}' -noprompt"
+KEYTOOL_DELETE_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + configDefaults.keytool_bin + " -delete -alias '{1}' -storepass '{2}' -noprompt"
KEYTOOL_KEYSTORE = " -keystore '{0}'"
-java_bin = "java"
-if OSCheck.is_windows_family():
- java_bin = "java.exe"
-
-SECURITY_PROVIDER_GET_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp {1}" +\
- os.pathsep + "{2} " +\
+SECURITY_PROVIDER_GET_CMD = "{0}" + os.sep + configDefaults.JAVA_EXE_SUBPATH + " -cp {1} " +\
"org.apache.ambari.server.security.encryption" +\
- ".CredentialProvider GET {3} {4} {5} " +\
- "> " + SERVER_OUT_FILE + " 2>&1"
+ ".CredentialProvider GET {2} {3} {4} " +\
+ "> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
-SECURITY_PROVIDER_PUT_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp {1}" +\
- os.pathsep + "{2} " +\
+SECURITY_PROVIDER_PUT_CMD = "{0}" + os.sep + configDefaults.JAVA_EXE_SUBPATH + " -cp {1} " +\
"org.apache.ambari.server.security.encryption" +\
- ".CredentialProvider PUT {3} {4} {5} " +\
- "> " + SERVER_OUT_FILE + " 2>&1"
+ ".CredentialProvider PUT {2} {3} {4} " +\
+ "> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
-SECURITY_PROVIDER_KEY_CMD = "{0}" + os.sep + "bin" + os.sep + java_bin + " -cp {1}" +\
- os.pathsep + "{2} " +\
+SECURITY_PROVIDER_KEY_CMD = "{0}" + os.sep + configDefaults.JAVA_EXE_SUBPATH + " -cp {1}" +\
"org.apache.ambari.server.security.encryption" +\
- ".MasterKeyServiceImpl {3} {4} {5} " +\
- "> " + SERVER_OUT_FILE + " 2>&1"
+ ".MasterKeyServiceImpl {2} {3} {4} " +\
+ "> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
SSL_KEY_DIR = 'security.server.keys_dir'
SSL_API_PORT = 'client.api.ssl.port'
@@ -521,7 +526,7 @@ def read_passwd_for_alias(alias, masterKey=""):
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
- "JDK manually to " + JDK_INSTALL_DIR)
+ "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
tempFileName = "ambari.passwd"
@@ -536,8 +541,7 @@ def read_passwd_for_alias(alias, masterKey=""):
if masterKey is None or masterKey == "":
masterKey = "None"
- command = SECURITY_PROVIDER_GET_CMD.format(jdk_path,
- get_conf_dir(), get_ambari_classpath(), alias, tempFilePath, masterKey)
+ command = SECURITY_PROVIDER_GET_CMD.format(jdk_path, get_full_ambari_classpath(), alias, tempFilePath, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider get passwd: " +
str(retcode))
@@ -574,14 +578,13 @@ def save_passwd_for_alias(alias, passwd, masterKey=""):
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
- "JDK manually to " + JDK_INSTALL_DIR)
+ "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
if masterKey is None or masterKey == "":
masterKey = "None"
- command = SECURITY_PROVIDER_PUT_CMD.format(jdk_path, get_conf_dir(),
- get_ambari_classpath(), alias, passwd, masterKey)
+ command = SECURITY_PROVIDER_PUT_CMD.format(jdk_path, get_full_ambari_classpath(), alias, passwd, masterKey)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider save passwd: " +
str(retcode))
@@ -597,7 +600,7 @@ def get_is_persisted(properties):
return (isPersisted, masterKeyFile)
def get_credential_store_location(properties):
- store_loc = properties[SECURITY_KEYS_DIR]
+ store_loc = get_value_from_properties(properties, SECURITY_KEYS_DIR, "")
if store_loc is None or store_loc == "":
store_loc = "/var/lib/ambari-server/keys/credentials.jceks"
else:
@@ -605,9 +608,9 @@ def get_credential_store_location(properties):
return store_loc
def get_master_key_location(properties):
- keyLocation = properties[SECURITY_MASTER_KEY_LOCATION]
+ keyLocation = get_value_from_properties(properties, SECURITY_MASTER_KEY_LOCATION, "")
if keyLocation is None or keyLocation == "":
- keyLocation = properties[SECURITY_KEYS_DIR]
+ keyLocation = get_value_from_properties(properties, SECURITY_KEYS_DIR, "")
return keyLocation
def get_original_master_key(properties):
@@ -679,10 +682,9 @@ def save_master_key(master_key, key_location, persist=True):
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
- "JDK manually to " + JDK_INSTALL_DIR)
+ "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
- command = SECURITY_PROVIDER_KEY_CMD.format(jdk_path,
- get_ambari_classpath(), get_conf_dir(), master_key, key_location, persist)
+ command = SECURITY_PROVIDER_KEY_CMD.format(jdk_path, get_full_ambari_classpath(), master_key, key_location, persist)
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from credential provider save KEY: " +
str(retcode))
@@ -722,20 +724,35 @@ def adjust_directory_permissions(ambari_user):
bootstrap_dir = get_value_from_properties(properties, BOOTSTRAP_DIR_PROPERTY)
print_info_msg("Cleaning bootstrap directory ({0}) contents...".format(bootstrap_dir))
shutil.rmtree(bootstrap_dir, True) #Ignore the non-existent dir error
- os.makedirs(bootstrap_dir)
+
+ #makedirs may fail with "Access denied" if executed right after rmtree. The only known good solution is a delayed retry.
+ retry_makedirs = 0
+ while True:
+ try:
+ os.makedirs(bootstrap_dir)
+ except OSError as e:
+ time.sleep(0.01) #Unnoticeable at this scale
+ retry_makedirs += 1
+ if retry_makedirs == 3:
+ raise
+ continue
+ break
+
+ ownership_list = configDefaults.NR_ADJUST_OWNERSHIP_LIST
+
# Add master key and credential store if exists
keyLocation = get_master_key_location(properties)
masterKeyFile = search_file(SECURITY_MASTER_KEY_FILENAME, keyLocation)
if masterKeyFile:
- NR_ADJUST_OWNERSHIP_LIST.append((masterKeyFile, MASTER_KEY_FILE_PERMISSIONS, "{0}", "{0}", False))
+ ownership_list.append((masterKeyFile, configDefaults.MASTER_KEY_FILE_PERMISSIONS, "{0}", "{0}", False))
credStoreFile = get_credential_store_location(properties)
if os.path.exists(credStoreFile):
- NR_ADJUST_OWNERSHIP_LIST.append((credStoreFile, CREDENTIALS_STORE_FILE_PERMISSIONS, "{0}", "{0}", False))
- trust_store_location = properties[SSL_TRUSTSTORE_PATH_PROPERTY]
+ ownership_list.append((credStoreFile, configDefaults.CREDENTIALS_STORE_FILE_PERMISSIONS, "{0}", "{0}", False))
+ trust_store_location = get_value_from_properties(properties, SSL_TRUSTSTORE_PATH_PROPERTY, None)
if trust_store_location:
- NR_ADJUST_OWNERSHIP_LIST.append((trust_store_location, TRUST_STORE_LOCATION_PERMISSIONS, "{0}", "{0}", False))
+ ownership_list.append((trust_store_location, configDefaults.TRUST_STORE_LOCATION_PERMISSIONS, "{0}", "{0}", False))
print "Adjusting ambari-server permissions and ownership..."
- for pack in NR_ADJUST_OWNERSHIP_LIST:
+ for pack in ownership_list:
file = pack[0]
mod = pack[1]
user = pack[2].format(ambari_user)
@@ -829,7 +846,7 @@ def setup_component_https(component, command, property, alias):
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
- "JDK manually to " + JDK_INSTALL_DIR
+ "JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
properties = get_ambari_properties()
@@ -979,7 +996,7 @@ def setup_master_key():
print_info_msg("Deleting master key file at location: " + str(
masterKeyFile))
except Exception, e:
- print 'ERROR: Could not remove master key file. %s' % e
+ print 'ERROR: Could not remove master key file. %s' % str(e)
# Blow up the credential store made with previous key, if any
store_file = get_credential_store_location(properties)
if os.path.exists(store_file):
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/userInput.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/userInput.py b/ambari-server/src/main/python/ambari_server/userInput.py
index 7a35831..ff4c0c7 100644
--- a/ambari-server/src/main/python/ambari_server/userInput.py
+++ b/ambari-server/src/main/python/ambari_server/userInput.py
@@ -22,7 +22,7 @@ import os
import getpass
import re
-from ambari_commons.logging_utils import *
+from ambari_commons.logging_utils import SILENT
#
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
index 7430ba1..f414d18 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
@@ -24,9 +24,10 @@ import os
import subprocess
import socket
-from ambari_commons import os_utils
from ambari_commons.os_check import OSCheck, OSConst
+from ambari_commons.os_utils import quote_path
from ambari_commons.inet_utils import download_file
+from ambari_server.serverConfiguration import configDefaults
from resource_management import Script, Execute, format
from ambari_agent.HostInfo import HostInfo
@@ -102,11 +103,8 @@ class CheckHost(Script):
java64_home = config['commandParams']['java_home']
print "Java home to check: " + java64_home
- java_bin = "java"
- if OSCheck.is_windows_family():
- java_bin = "java.exe"
-
- if not os.path.isfile(os.path.join(java64_home, "bin", java_bin)):
+
+ if not os.path.isfile(os.path.join(java64_home, configDefaults.JAVA_EXE_SUBPATH)):
print "Java home doesn't exist!"
java_home_check_structured_output = {"exit_code" : 1, "message": "Java home doesn't exist!"}
else:
@@ -152,13 +150,9 @@ class CheckHost(Script):
jdbc_path = os.path.join(agent_cache_dir, jdbc_name)
check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name)
- java_bin = "java"
- class_path_delimiter = ":"
- if OSCheck.is_windows_family():
- java_bin = "java.exe"
- class_path_delimiter = ";"
+ class_path_delimiter = os.pathsep
- java_exec = os.path.join(java64_home, "bin",java_bin)
+ java_exec = os.path.join(java64_home, configDefaults.JAVA_EXE_SUBPATH)
if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
@@ -172,7 +166,7 @@ class CheckHost(Script):
# download and install java if it doesn't exists
if not os.path.isfile(java_exec):
jdk_name = config['commandParams']['jdk_name']
- jdk_url = "{}/{}".format(jdk_location, jdk_name)
+ jdk_url = "{0}/{1}".format(jdk_location, jdk_name)
jdk_download_target = os.path.join(agent_cache_dir, jdk_name)
java_dir = os.path.dirname(java64_home)
try:
@@ -192,8 +186,8 @@ class CheckHost(Script):
install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_download_target} > /dev/null 2>&1")
install_path = ["/bin","/usr/bin/"]
elif jdk_name.endswith(".exe"):
- install_cmd = "{} /s INSTALLDIR={} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
- os_utils.quote_path(jdk_download_target), os_utils.quote_path(java64_home),
+ install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
+ quote_path(jdk_download_target), quote_path(java64_home),
)
install_path = [java_dir]
[4/7] ambari git commit: AMBARI-8373 Refactor the OS-dependent Ambari
Server Windows components
Posted by fb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/serverConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
index aab1167..d8b93dd 100644
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
@@ -20,18 +20,23 @@ limitations under the License.
import datetime
import glob
+import os
import re
-
-from ambari_commons.os_utils import *
+import shutil
+import string
+import tempfile
+from ambari_commons import OSCheck, OSConst
+from ambari_commons.exceptions import FatalException
+from ambari_commons.os_family_impl import OsFamilyImpl
+
+from ambari_commons.os_utils import search_file, run_os_command
from ambari_commons.logging_utils import print_warning_msg, print_info_msg, print_error_msg
-from properties import Properties
+from ambari_server.properties import Properties
-if OSCheck.is_windows_family():
- from serverConfiguration_windows import *
-else:
- # MacOS not supported
- from serverConfiguration_linux import *
+OS_VERSION = OSCheck().get_os_major_version()
+OS_TYPE = OSCheck.get_os_type()
+OS_FAMILY = OSCheck.get_os_family()
# Non-root user setup commands
NR_USER_PROPERTY = "ambari-server.user"
@@ -70,6 +75,9 @@ JAVA_HOME_PROPERTY = "java.home"
JDK_NAME_PROPERTY = "jdk.name"
JCE_NAME_PROPERTY = "jce.name"
+DEFAULT_JDK16_LOCATION = "/usr/jdk64/jdk1.6.0_31"
+JDK_NAMES = ["jdk-7u67-linux-x64.tar.gz", "jdk-6u31-linux-x64.bin"]
+
#JCE Policy files
JCE_POLICY_FILENAMES = ["UnlimitedJCEPolicyJDK7.zip", "jce_policy-6.zip"]
JCE_DOWNLOAD_CMD = "curl -o {0} {1}"
@@ -78,10 +86,11 @@ JCE_MIN_FILESIZE = 5000
# JDBC
#TODO property used incorrectly in local case, it was meant to be dbms name, not postgres database name,
# has workaround for now, as we don't need dbms name if persistence_type=local
-JDBC_DATABASE_PROPERTY = "server.jdbc.database"
+JDBC_DATABASE_PROPERTY = "server.jdbc.database" # E.g., embedded|oracle|mysql|postgres|sqlserver
+JDBC_DATABASE_NAME_PROPERTY = "server.jdbc.database_name" # E.g., ambari. Not used on Windows.
JDBC_HOSTNAME_PROPERTY = "server.jdbc.hostname"
JDBC_PORT_PROPERTY = "server.jdbc.port"
-JDBC_SCHEMA_PROPERTY = "server.jdbc.schema"
+JDBC_POSTGRES_SCHEMA_PROPERTY = "server.jdbc.postgres.schema" # Only for postgres, defaults to same value as DB name
JDBC_USER_NAME_PROPERTY = "server.jdbc.user.name"
JDBC_PASSWORD_PROPERTY = "server.jdbc.user.passwd"
@@ -91,6 +100,8 @@ JDBC_RCA_PASSWORD_FILENAME = "rca_password.dat"
CLIENT_API_PORT_PROPERTY = "client.api.port"
CLIENT_API_PORT = "8080"
+SERVER_VERSION_FILE_PATH = "server.version.file"
+
PERSISTENCE_TYPE_PROPERTY = "server.persistence.type"
JDBC_DRIVER_PROPERTY = "server.jdbc.driver"
JDBC_DRIVER_PATH_PROPERTY = "server.jdbc.driver.path"
@@ -108,6 +119,29 @@ JDBC_RCA_PASSWORD_FILE_PROPERTY = "server.jdbc.rca.user.passwd"
JDBC_RCA_PASSWORD_ALIAS = "ambari.db.password"
+### # Windows-specific # ###
+JDBC_USE_INTEGRATED_AUTH_PROPERTY = "server.jdbc.use.integrated.auth"
+
+JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY = "server.jdbc.rca.use.integrated.auth"
+
+JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY = "scom.sink.db.use.integrated.auth"
+
+METRICS_PERSISTENCE_TYPE_PROPERTY = "metrics.persistence.type"
+
+JDBC_METRICS_DATABASE_PROPERTY = "scom.sink.db.database"
+JDBC_METRICS_HOSTNAME_PROPERTY = "scom.sink.db.hostname"
+JDBC_METRICS_PORT_PROPERTY = "scom.sink.db.port"
+JDBC_METRICS_SCHEMA_PROPERTY = "scom.sink.db.schema"
+
+JDBC_METRICS_DRIVER_PROPERTY = "scom.sink.db.driver"
+JDBC_METRICS_URL_PROPERTY = "scom.sink.db.url"
+JDBC_METRICS_USER_NAME_PROPERTY = "scom.sink.db.username"
+JDBC_METRICS_PASSWORD_PROPERTY = "scom.sink.db.password"
+JDBC_METRICS_PASSWORD_FILENAME = "scom_password.dat"
+
+JDBC_METRICS_PASSWORD_ALIAS = "scom.db.password"
+### # End Windows-specific # ###
+
# resources repo configuration
RESOURCES_DIR_PROPERTY = "resources.dir"
RESOURCES_DIR_DEFAULT = "resources"
@@ -119,16 +153,166 @@ STACK_LOCATION_DEFAULT = "resources" + os.sep + "stacks"
# JDK
JDK_RELEASES="java.releases"
+# Views
+VIEWS_DIR_PROPERTY = "views.dir"
+
# configuration backup
back_up_file_path = None
+class ServerConfigDefaults(object):
+ def __init__(self):
+ self.JAVA_SHARE_PATH = "/usr/share/java"
+ self.OUT_DIR = os.sep + os.path.join("var", "log", "ambari-server")
+ self.SERVER_OUT_FILE = os.path.join(self.OUT_DIR, "ambari-server.out")
+ self.SERVER_LOG_FILE = os.path.join(self.OUT_DIR, "ambari-server.log")
+ self.ROOT_FS_PATH = os.sep
+
+ self.JDK_INSTALL_DIR = ""
+ self.JDK_SEARCH_PATTERN = ""
+ self.JAVA_EXE_SUBPATH = ""
+
+ # Configuration defaults
+ self.DEFAULT_CONF_DIR = ""
+ self.PID_DIR = os.sep + os.path.join("var", "run", "ambari-server")
+ self.DEFAULT_LIBS_DIR = ""
+
+ # ownership/permissions mapping
+ # path - permissions - user - group - recursive
+ # Rules are executed in the same order as they are listed
+ # {0} in user/group will be replaced by customized ambari-server username
+ self.NR_ADJUST_OWNERSHIP_LIST = []
+
+ self.MASTER_KEY_FILE_PERMISSIONS = "600"
+ self.CREDENTIALS_STORE_FILE_PERMISSIONS = "600"
+ self.TRUST_STORE_LOCATION_PERMISSIONS = "600"
+
+ self.DEFAULT_DB_NAME = "ambari"
+
+ self.DEFAULT_VIEWS_DIR = ""
+
+ #keytool commands
+ self.keytool_bin = ""
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class ServerConfigDefaultsWindows(ServerConfigDefaults):
+ def __init__(self):
+ super(ServerConfigDefaultsWindows, self).__init__()
+ self.JDK_INSTALL_DIR = "C:\\"
+ self.JDK_SEARCH_PATTERN = "j[2se|dk|re]*"
+ self.JAVA_EXE_SUBPATH = "bin\\java.exe"
+
+ # Configuration defaults
+ self.DEFAULT_CONF_DIR = "conf"
+ self.DEFAULT_LIBS_DIR = "lib"
+
+ # ownership/permissions mapping
+ # path - permissions - user - group - recursive
+ # Rules are executed in the same order as they are listed
+ # {0} in user/group will be replaced by customized ambari-server username
+ # The permissions are icacls
+ self.NR_ADJUST_OWNERSHIP_LIST = [
+ (self.OUT_DIR, "M", "{0}", True), #0110-0100-0100 rw-r-r
+ (self.OUT_DIR, "F", "{0}", False), #0111-0101-0101 rwx-rx-rx
+ (self.PID_DIR, "M", "{0}", True),
+ (self.PID_DIR, "F", "{0}", False),
+ ("bootstrap", "F", "{0}", False),
+ ("ambari-env.cmd", "F", "{0}", False),
+ ("keystore", "M", "{0}", True),
+ ("keystore", "F", "{0}", False),
+ ("keystore\\db", "700", "{0}", False),
+ ("keystore\\db\\newcerts", "700", "{0}", False),
+ ("resources\\stacks", "755", "{0}", True),
+ ("resources\\custom_actions", "755", "{0}", True),
+ ("conf", "644", "{0}", True),
+ ("conf", "755", "{0}", False),
+ ("conf\\password.dat", "640", "{0}", False),
+ # Also, /etc/ambari-server/conf/password.dat
+ # is generated later at store_password_file
+ ]
+
+ self.DEFAULT_VIEWS_DIR = "resources\\views"
+
+ #keytool commands
+ self.keytool_bin = "keytool.exe"
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class ServerConfigDefaultsLinux(ServerConfigDefaults):
+ def __init__(self):
+ super(ServerConfigDefaultsLinux, self).__init__()
+ # JDK
+ self.JDK_INSTALL_DIR = "/usr/jdk64"
+ self.JDK_SEARCH_PATTERN = "jdk*"
+ self.JAVA_EXE_SUBPATH = "bin/java"
+
+ # Configuration defaults
+ self.DEFAULT_CONF_DIR = "/etc/ambari-server/conf"
+ self.DEFAULT_LIBS_DIR = "/usr/lib/ambari-server"
+
+ # ownership/permissions mapping
+ # path - permissions - user - group - recursive
+ # Rules are executed in the same order as they are listed
+ # {0} in user/group will be replaced by customized ambari-server username
+ self.NR_ADJUST_OWNERSHIP_LIST = [
+ ("/var/log/ambari-server", "644", "{0}", True),
+ ("/var/log/ambari-server", "755", "{0}", False),
+ ("/var/run/ambari-server", "644", "{0}", True),
+ ("/var/run/ambari-server", "755", "{0}", False),
+ ("/var/run/ambari-server/bootstrap", "755", "{0}", False),
+ ("/var/lib/ambari-server/ambari-env.sh", "700", "{0}", False),
+ ("/var/lib/ambari-server/keys", "600", "{0}", True),
+ ("/var/lib/ambari-server/keys", "700", "{0}", False),
+ ("/var/lib/ambari-server/keys/db", "700", "{0}", False),
+ ("/var/lib/ambari-server/keys/db/newcerts", "700", "{0}", False),
+ ("/var/lib/ambari-server/keys/.ssh", "700", "{0}", False),
+ ("/var/lib/ambari-server/resources/stacks/", "755", "{0}", True),
+ ("/var/lib/ambari-server/resources/custom_actions/", "755", "{0}", True),
+ ("/var/lib/ambari-server/resources/host_scripts/", "755", "{0}", True),
+ ("/var/lib/ambari-server/resources/views", "644", "{0}", True),
+ ("/var/lib/ambari-server/resources/views", "755", "{0}", False),
+ ("/var/lib/ambari-server/resources/views/work", "755", "{0}", True),
+ ("/etc/ambari-server/conf", "644", "{0}", True),
+ ("/etc/ambari-server/conf", "755", "{0}", False),
+ ("/etc/ambari-server/conf/password.dat", "640", "{0}", False),
+ ("/var/lib/ambari-server/keys/pass.txt", "640", "{0}", False),
+ ("/etc/ambari-server/conf/ldap-password.dat", "640", "{0}", False),
+ ("/var/run/ambari-server/stack-recommendations/", "644", "{0}", True),
+ ("/var/run/ambari-server/stack-recommendations/", "755", "{0}", False),
+ ("/var/lib/ambari-server/data/tmp/", "644", "{0}", True),
+ ("/var/lib/ambari-server/data/tmp/", "755", "{0}", False),
+ # Also, /etc/ambari-server/conf/password.dat
+ # is generated later at store_password_file
+ ]
+
+ self.DEFAULT_VIEWS_DIR = "/var/lib/ambari-server/resources/views"
+
+ #keytool commands
+ self.keytool_bin = "keytool"
+
+configDefaults = ServerConfigDefaults()
+
+
+SCHEMA_UPGRADE_HELPER_CMD = "{0}" + os.sep + "bin" + os.sep + "java -cp {1} " + \
+ "org.apache.ambari.server.upgrade.SchemaUpgradeHelper" + \
+ " > " + configDefaults.SERVER_OUT_FILE + " 2>&1"
+
+VIEW_EXTRACT_CMD = "{0}" + os.sep + "bin" + os.sep + "java -cp {1} " + \
+ "org.apache.ambari.server.view.ViewRegistry extract {2} " + \
+ "> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
+
+
+#Common setup or upgrade message
+SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server setup\" command to create the user\n" \
+ "- If this is an upgrade of an existing setup, run the \"ambari-server upgrade\" command.\n" \
+ "Refer to the Ambari documentation for more information on setup and upgrade."
+
+
def get_conf_dir():
try:
conf_dir = os.environ[AMBARI_CONF_VAR]
return conf_dir
except KeyError:
- default_conf_dir = DEFAULT_CONF_DIR
+ default_conf_dir = configDefaults.DEFAULT_CONF_DIR
print AMBARI_CONF_VAR + " is not set, using default " + default_conf_dir
return default_conf_dir
@@ -222,8 +406,8 @@ def update_database_name_property():
err = "Error getting ambari properties"
raise FatalException(-1, err)
print_warning_msg(JDBC_DATABASE_PROPERTY + " property isn't set in " +
- AMBARI_PROPERTIES_FILE + ". Setting it to default value - " + DEFAULT_DB_NAME)
- properties.process_pair(JDBC_DATABASE_PROPERTY, DEFAULT_DB_NAME)
+ AMBARI_PROPERTIES_FILE + ". Setting it to default value - " + configDefaults.DEFAULT_DB_NAME)
+ properties.process_pair(JDBC_DATABASE_PROPERTY, configDefaults.DEFAULT_DB_NAME)
conf_file = find_properties_file()
try:
properties.store(open(conf_file, "w"))
@@ -249,23 +433,22 @@ def parse_properties_file(args):
print_error_msg("Error getting ambari properties")
return -1
- # args.server_version_file_path = properties[SERVER_VERSION_FILE_PATH]
+ args.server_version_file_path = properties[SERVER_VERSION_FILE_PATH]
args.persistence_type = properties[PERSISTENCE_TYPE_PROPERTY]
args.jdbc_url = properties[JDBC_URL_PROPERTY]
+ args.dbms = properties[JDBC_DATABASE_PROPERTY]
if not args.persistence_type:
args.persistence_type = "local"
if args.persistence_type == 'remote':
- args.dbms = properties[JDBC_DATABASE_PROPERTY]
args.database_host = properties[JDBC_HOSTNAME_PROPERTY]
args.database_port = properties[JDBC_PORT_PROPERTY]
- args.database_name = properties[JDBC_SCHEMA_PROPERTY]
- else:
- #TODO incorrect property used!! leads to bunch of troubles. Workaround for now
- args.database_name = properties[JDBC_DATABASE_PROPERTY]
+ args.database_name = properties[JDBC_DATABASE_NAME_PROPERTY]
args.database_username = properties[JDBC_USER_NAME_PROPERTY]
+ args.postgres_schema = properties[JDBC_POSTGRES_SCHEMA_PROPERTY] \
+ if JDBC_POSTGRES_SCHEMA_PROPERTY in properties.propertyNames() else None
args.database_password_file = properties[JDBC_PASSWORD_PROPERTY]
if args.database_password_file:
if not is_alias_string(args.database_password_file):
@@ -280,9 +463,9 @@ def run_schema_upgrade():
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
- "JDK manually to " + JDK_INSTALL_DIR)
+ "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
- command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_conf_dir(), get_ambari_classpath())
+ command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath())
(retcode, stdout, stderr) = run_os_command(command)
print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
if retcode > 0:
@@ -424,7 +607,10 @@ def remove_property(key):
# Checks if options determine local DB configuration
#
def is_local_database(args):
- return hasattr(args, 'persistence_type') and args.persistence_type == 'local'
+ try:
+ return args.persistence_type == 'local'
+ except AttributeError:
+ return False
#
### JDK ###
@@ -434,15 +620,7 @@ def is_local_database(args):
# Describes the JDK configuration data, necessary for download and installation
#
class JDKRelease:
- name = ""
- desc = ""
- url = ""
- dest_file = ""
- jcpol_url = "http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip"
- dest_jcpol_file = ""
- inst_dir = ""
-
- def __init__(self, i_name, i_desc, i_url, i_dest_file, i_jcpol_url, i_dest_jcpol_file, i_inst_dir):
+ def __init__(self, i_name, i_desc, i_url, i_dest_file, i_jcpol_url, i_dest_jcpol_file, i_inst_dir, i_reg_exp):
if i_name is None or i_name is "":
raise FatalException(-1, "Invalid JDK name: " + (i_desc or ""))
self.name = i_name
@@ -464,14 +642,17 @@ class JDKRelease:
else:
self.dest_jcpol_file = i_dest_jcpol_file
if i_inst_dir is None or i_inst_dir is "":
- self.inst_dir = "C:\\" + i_desc
+ self.inst_dir = os.path.join(configDefaults.JDK_INSTALL_DIR, i_desc)
else:
self.inst_dir = i_inst_dir
+ if i_reg_exp is None or i_reg_exp is "":
+ raise FatalException(-1, "Invalid output parsing regular expression for JDK " + i_name)
+ self.reg_exp = i_reg_exp
@classmethod
def from_properties(cls, properties, section_name):
- (desc, url, dest_file, jcpol_url, jcpol_file, inst_dir) = JDKRelease.__load_properties(properties, section_name)
- cls = JDKRelease(section_name, desc, url, dest_file, jcpol_url, jcpol_file, inst_dir)
+ (desc, url, dest_file, jcpol_url, jcpol_file, inst_dir, reg_exp) = JDKRelease.__load_properties(properties, section_name)
+ cls = JDKRelease(section_name, desc, url, dest_file, jcpol_url, jcpol_file, inst_dir, reg_exp)
return cls
@staticmethod
@@ -485,6 +666,9 @@ class JDKRelease:
if not properties.has_key(section_name + ".url"):
raise FatalException(-1, "Invalid JDK URL in the properties section: " + section_name)
url = properties[section_name + ".url"] #Required
+ if not properties.has_key(section_name + ".re"):
+ raise FatalException(-1, "Invalid JDK output parsing regular expression in the properties section: " + section_name)
+ reg_exp = properties[section_name + ".re"] #Required
if(properties.has_key(section_name + ".dest-file")): #Not critical
dest_file = properties[section_name + ".dest-file"]
else:
@@ -500,8 +684,8 @@ class JDKRelease:
if(properties.has_key(section_name + ".home")): #Not critical
inst_dir = properties[section_name + ".home"]
else:
- inst_dir = "C:\\" + section_name
- return (desc, url, dest_file, jcpol_url, jcpol_file, inst_dir)
+ inst_dir = os.path.join(configDefaults.JDK_INSTALL_DIR, section_name)
+ return (desc, url, dest_file, jcpol_url, jcpol_file, inst_dir, reg_exp)
pass
def get_ambari_jars():
@@ -509,7 +693,7 @@ def get_ambari_jars():
conf_dir = os.environ[AMBARI_SERVER_LIB]
return conf_dir
except KeyError:
- default_jar_location = DEFAULT_LIBS_DIR
+ default_jar_location = configDefaults.DEFAULT_LIBS_DIR
print_info_msg(AMBARI_SERVER_LIB + " is not set, using default "
+ default_jar_location)
return default_jar_location
@@ -517,8 +701,8 @@ def get_ambari_jars():
def get_share_jars():
share_jars = ""
file_list = []
- file_list.extend(glob.glob(JAVA_SHARE_PATH + os.sep + "*mysql*"))
- file_list.extend(glob.glob(JAVA_SHARE_PATH + os.sep + "*ojdbc*"))
+ file_list.extend(glob.glob(configDefaults.JAVA_SHARE_PATH + os.sep + "*mysql*"))
+ file_list.extend(glob.glob(configDefaults.JAVA_SHARE_PATH + os.sep + "*ojdbc*"))
if len(file_list) > 0:
share_jars = string.join(file_list, os.pathsep)
return share_jars
@@ -540,6 +724,15 @@ def get_ambari_classpath():
ambari_cp = ambari_cp + os.pathsep + share_cp
return ambari_cp
+def get_full_ambari_classpath(conf_dir = None):
+ if conf_dir is None:
+ conf_dir = get_conf_dir()
+ cp = conf_dir + os.pathsep + get_ambari_classpath()
+ if cp.find(' ') != -1:
+ cp = '"' + cp + '"'
+ return cp
+
+
def get_JAVA_HOME():
properties = get_ambari_properties()
if properties == -1:
@@ -559,7 +752,7 @@ def get_JAVA_HOME():
def validate_jdk(jdk_path):
if jdk_path:
if os.path.exists(jdk_path):
- java_exe_path = os.path.join(jdk_path, JAVA_EXE_SUBPATH)
+ java_exe_path = os.path.join(jdk_path, configDefaults.JAVA_EXE_SUBPATH)
if os.path.exists(java_exe_path) and os.path.isfile(java_exe_path):
return True
return False
@@ -572,8 +765,8 @@ def find_jdk():
if jdkPath:
if validate_jdk(jdkPath):
return jdkPath
- print "Looking for available JDKs at " + JDK_INSTALL_DIR
- jdks = glob.glob(JDK_INSTALL_DIR + os.sep + JDK_SEARCH_PATTERN)
+ print "Looking for available JDKs at " + configDefaults.JDK_INSTALL_DIR
+ jdks = glob.glob(configDefaults.JDK_INSTALL_DIR + os.sep + configDefaults.JDK_SEARCH_PATTERN)
#[fbarca] Use the newest JDK
jdks.sort(None, None, True)
print "Found: " + str(jdks)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/serverConfiguration_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration_linux.py b/ambari-server/src/main/python/ambari_server/serverConfiguration_linux.py
deleted file mode 100644
index a21437a..0000000
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration_linux.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-JAVA_SHARE_PATH = "/usr/share/java"
-OUT_DIR = "/var/log/ambari-server"
-SERVER_OUT_FILE = OUT_DIR + "/ambari-server.out"
-SERVER_LOG_FILE = OUT_DIR + "/ambari-server.log"
-ROOT_FS_PATH = "/"
-
-# JDK
-JDK_NAMES = ["jdk-7u45-linux-x64.tar.gz", "jdk-6u31-linux-x64.bin"]
-DEFAULT_JDK16_LOCATION = "/usr/jdk64/jdk1.6.0_31"
-JDK_INSTALL_DIR = "/usr/jdk64"
-JDK_SEARCH_PATTERN = "jdk*"
-JAVA_EXE_SUBPATH = "bin/java"
-
-# Configuration defaults
-DEFAULT_CONF_DIR = "/etc/ambari-server/conf"
-PID_DIR = "/var/run/ambari-server"
-DEFAULT_LIBS_DIR = "/usr/lib/ambari-server"
-
-# ownership/permissions mapping
-# path - permissions - user - group - recursive
-# Rules are executed in the same order as they are listed
-# {0} in user/group will be replaced by customized ambari-server username
-NR_ADJUST_OWNERSHIP_LIST = [
-
- ("/var/log/ambari-server", "644", "{0}", True),
- ("/var/log/ambari-server", "755", "{0}", False),
- ("/var/run/ambari-server", "644", "{0}", True),
- ("/var/run/ambari-server", "755", "{0}", False),
- ("/var/run/ambari-server/bootstrap", "755", "{0}", False),
- ("/var/lib/ambari-server/ambari-env.sh", "700", "{0}", False),
- ("/var/lib/ambari-server/keys", "600", "{0}", True),
- ("/var/lib/ambari-server/keys", "700", "{0}", False),
- ("/var/lib/ambari-server/keys/db", "700", "{0}", False),
- ("/var/lib/ambari-server/keys/db/newcerts", "700", "{0}", False),
- ("/var/lib/ambari-server/keys/.ssh", "700", "{0}", False),
- ("/var/lib/ambari-server/resources/stacks/", "755", "{0}", True),
- ("/var/lib/ambari-server/resources/custom_actions/", "755", "{0}", True),
- ("/etc/ambari-server/conf", "644", "{0}", True),
- ("/etc/ambari-server/conf", "755", "{0}", False),
- ("/etc/ambari-server/conf/password.dat", "640", "{0}", False),
- # Also, /etc/ambari-server/conf/password.dat
- # is generated later at store_password_file
-]
-
-MASTER_KEY_FILE_PERMISSIONS = "600"
-CREDENTIALS_STORE_FILE_PERMISSIONS = "600"
-TRUST_STORE_LOCATION_PERMISSIONS = "600"
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/serverConfiguration_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration_windows.py b/ambari-server/src/main/python/ambari_server/serverConfiguration_windows.py
deleted file mode 100644
index a0fa508..0000000
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration_windows.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import string
-import os
-
-JDBC_USE_INTEGRATED_AUTH_PROPERTY = "server.jdbc.use.integrated.auth"
-
-JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY = "server.jdbc.rca.use.integrated.auth"
-
-JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY = "scom.sink.db.use.integrated.auth"
-
-METRICS_PERSISTENCE_TYPE_PROPERTY = "metrics.persistence.type"
-
-JDBC_METRICS_DATABASE_PROPERTY = "scom.sink.db.database"
-JDBC_METRICS_HOSTNAME_PROPERTY = "scom.sink.db.hostname"
-JDBC_METRICS_PORT_PROPERTY = "scom.sink.db.port"
-JDBC_METRICS_SCHEMA_PROPERTY = "scom.sink.db.schema"
-
-JDBC_METRICS_DRIVER_PROPERTY = "scom.sink.db.driver"
-JDBC_METRICS_URL_PROPERTY = "scom.sink.db.url"
-JDBC_METRICS_USER_NAME_PROPERTY = "scom.sink.db.username"
-JDBC_METRICS_PASSWORD_PROPERTY = "scom.sink.db.password"
-JDBC_METRICS_PASSWORD_FILENAME = "scom_password.dat"
-
-JDBC_METRICS_PASSWORD_ALIAS = "scom.db.password"
-
-JAVA_SHARE_PATH = "/usr/share/java"
-OUT_DIR = "\\var\\log\\ambari-server"
-SERVER_OUT_FILE = OUT_DIR + "\\ambari-server.out"
-SERVER_LOG_FILE = OUT_DIR + "\\ambari-server.log"
-ROOT_FS_PATH = "\\"
-
-JDK_INSTALL_DIR = "C:\\"
-JDK_SEARCH_PATTERN = "j[2se|dk|re]*"
-JAVA_EXE_SUBPATH = "bin\\java.exe"
-
-# Configuration defaults
-DEFAULT_CONF_DIR = "conf"
-PID_DIR = "\\var\\run\\ambari-server"
-DEFAULT_LIBS_DIR = "lib"
-
-# ownership/permissions mapping
-# path - permissions - user - group - recursive
-# Rules are executed in the same order as they are listed
-# {0} in user/group will be replaced by customized ambari-server username
-# The permissions are icacls
-NR_ADJUST_OWNERSHIP_LIST = [
-
- (OUT_DIR, "M", "{0}", True), #0110-0100-0100 rw-r-r
- (OUT_DIR, "F", "{0}", False), #0111-0101-0101 rwx-rx-rx
- (PID_DIR, "M", "{0}", True),
- (PID_DIR, "F", "{0}", False),
- ("bootstrap", "F", "{0}", False),
- ("ambari-env.cmd", "F", "{0}", False),
- ("keystore", "M", "{0}", True),
- ("keystore", "F", "{0}", False),
- ("keystore\\db", "700", "{0}", False),
- ("keystore\\db\\newcerts", "700", "{0}", False),
- ("resources\\stacks", "755", "{0}", True),
- ("resources\\custom_actions", "755", "{0}", True),
- ("conf", "644", "{0}", True),
- ("conf", "755", "{0}", False),
- ("conf\\password.dat", "640", "{0}", False),
- # Also, conf\password.dat
- # is generated later at store_password_file
-]
-
-MASTER_KEY_FILE_PERMISSIONS = "600"
-CREDENTIALS_STORE_FILE_PERMISSIONS = "600"
-TRUST_STORE_LOCATION_PERMISSIONS = "600"
-
-SCHEMA_UPGRADE_HELPER_CMD = "{0}" + os.sep + "bin" + os.sep + "java -cp {1}" + \
- os.pathsep + "{2} " + \
- "org.apache.ambari.server.upgrade.SchemaUpgradeHelper" + \
- " > " + SERVER_OUT_FILE + " 2>&1"
-
-STACK_UPGRADE_HELPER_CMD = "{0}" + os.sep + "bin" + os.sep + "java -cp {1}" + \
- os.pathsep + "{2} " + \
- "org.apache.ambari.server.upgrade.StackUpgradeHelper" + \
- " {3} {4} > " + SERVER_OUT_FILE + " 2>&1"
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/serverSetup.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py b/ambari-server/src/main/python/ambari_server/serverSetup.py
index 37e46f5..17b7aed 100644
--- a/ambari-server/src/main/python/ambari_server/serverSetup.py
+++ b/ambari-server/src/main/python/ambari_server/serverSetup.py
@@ -17,30 +17,65 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
-
-import socket
+import optparse
+import os
+import platform
+import re
+import shutil
import sys
-import urllib2
-from ambari_commons.inet_utils import force_download_file
-from ambari_commons.logging_utils import print_warning_msg, print_error_msg
-
-from serverConfiguration import *
-from setupSecurity import adjust_directory_permissions, get_is_secure, store_password_file, encrypt_password, \
- get_is_persisted
-from userInput import *
-from utils import *
-if OSCheck.is_windows_family():
- from serverSetup_windows import *
-else:
- # MacOS not supported
- from serverSetup_linux import *
-
-OS_VERSION = OSCheck().get_os_major_version()
-OS_TYPE = OSCheck.get_os_type()
-OS_FAMILY = OSCheck.get_os_family()
+from ambari_commons import OSCheck, OSConst, Firewall
+from ambari_commons.exceptions import FatalException
+from ambari_commons.inet_utils import force_download_file
+from ambari_commons.logging_utils import print_error_msg, print_info_msg, print_warning_msg, SILENT
+from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl
+from ambari_commons.os_utils import is_root, run_os_command
+from ambari_commons.str_utils import compress_backslashes
+from ambari_server.dbConfiguration import DBMSConfigFactory
+from ambari_server.serverConfiguration import get_ambari_properties, read_ambari_user, update_database_name_property, \
+ parse_properties_file, run_schema_upgrade, update_ambari_properties, update_properties, write_property, \
+ get_full_ambari_classpath, get_JAVA_HOME, find_jdk, \
+ configDefaults, JDKRelease, \
+ OS_VERSION, OS_TYPE, OS_FAMILY, \
+ NR_USER_PROPERTY, OS_TYPE_PROPERTY, AMBARI_PROPERTIES_FILE, \
+ JAVA_HOME, JAVA_HOME_PROPERTY, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY, \
+ JDBC_URL_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_METRICS_URL_PROPERTY, \
+ RESOURCES_DIR_PROPERTY, JDK_RELEASES, VIEWS_DIR_PROPERTY, VIEW_EXTRACT_CMD, SETUP_OR_UPGRADE_MSG
+from ambari_server.setupSecurity import get_is_secure, get_is_persisted, adjust_directory_permissions
+from ambari_server.userInput import get_YN_input, get_validated_string_input
+from ambari_server.utils import locate_file
+
+### # Linux-specific # ###
+# selinux commands
+GET_SE_LINUX_ST_CMD = locate_file('sestatus', '/usr/sbin')
+SE_SETENFORCE_CMD = "setenforce 0"
+SE_STATUS_DISABLED = "disabled"
+SE_STATUS_ENABLED = "enabled"
+SE_MODE_ENFORCING = "enforcing"
+SE_MODE_PERMISSIVE = "permissive"
+### # End Linux-specific # ###
+
+### # Windows-specific # ###
+SERVICE_USERNAME_KEY = "TMP_AMBARI_USERNAME"
+SERVICE_PASSWORD_KEY = "TMP_AMBARI_PASSWORD"
+### # End Windows-specific # ###
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def get_message_error_not_root():
+ return 'Ambari-server setup should be run with administrator-level privileges'
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def get_message_error_not_root():
+ return 'Ambari-server setup should be run with root-level privileges'
+
+MESSAGE_ERROR_NOT_ROOT = get_message_error_not_root()
+
+def get_supported_dbms():
+ factory = DBMSConfigFactory()
+ return factory.get_supported_dbms()
+
+JDBC_DB_OPTION_VALUES = get_supported_dbms()
-JDK_INDEX = 0
def verify_setup_allowed():
properties = get_ambari_properties()
@@ -58,260 +93,595 @@ def verify_setup_allowed():
return 0
-def check_ambari_user():
- try:
- user = read_ambari_user()
- create_user = False
- update_user_setting = False
- if user is not None:
- create_user = get_YN_input(NR_USER_CHANGE_PROMPT.format(user), False)
- update_user_setting = create_user # Only if we will create another user
- else: # user is not configured yet
- update_user_setting = True # Write configuration anyway
- create_user = get_YN_input(NR_USER_CUSTOMIZE_PROMPT, False)
- if not create_user:
- user = NR_DEFAULT_USER
-
- if create_user:
- (retcode, user) = create_custom_user()
- if retcode != 0:
- return retcode
-
- if update_user_setting:
- write_property(NR_USER_PROPERTY, user)
+#
+# Security enhancements (Linux only)
+#
- adjust_directory_permissions(user)
- except OSError as e:
- print_error_msg("Failed: %s" % e.strerror)
- return 4
- except Exception as e:
- print_error_msg("Unexpected error %s" % e)
- return 1
+#
+# Checks SELinux
+#
+def _check_selinux():
+ try:
+ retcode, out, err = run_os_command(GET_SE_LINUX_ST_CMD)
+ se_status = re.search('(disabled|enabled)', out).group(0)
+ print "SELinux status is '" + se_status + "'"
+ if se_status == SE_STATUS_DISABLED:
+ return 0
+ else:
+ try:
+ se_mode = re.search('(enforcing|permissive)', out).group(0)
+ except AttributeError:
+ err = "Error determining SELinux mode. Exiting."
+ raise FatalException(1, err)
+ print "SELinux mode is '" + se_mode + "'"
+ if se_mode == SE_MODE_ENFORCING:
+ print "Temporarily disabling SELinux"
+ run_os_command(SE_SETENFORCE_CMD)
+ print_warning_msg(
+ "SELinux is set to 'permissive' mode and temporarily disabled.")
+ ok = get_YN_input("OK to continue [y/n] (y)? ", True)
+ if not ok:
+ raise FatalException(1, None)
+ return 0
+ except OSError:
+ print_warning_msg("Could not run {0}: OK".format(GET_SE_LINUX_ST_CMD))
return 0
-def create_custom_user():
- return os_create_custom_user()
+# No security enhancements in Windows
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def disable_security_enhancements():
+ retcode = 0
+ err = ''
+ return (retcode, err)
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def disable_security_enhancements():
+ print 'Checking SELinux...'
+ err = ''
+ retcode = _check_selinux()
+ if not retcode == 0:
+ err = 'Failed to disable SELinux. Exiting.'
+ return (retcode, err)
-# ## JDK ###
-
#
-# Downloads and installs the JDK and the JCE policy archive
+# User account creation
#
-def _dowload_jdk(jdk_url, dest_file):
- jdk_download_fail_msg = " Failed to download JDK: {0}. Please check that Oracle " \
- "JDK is available at {1}. Also you may specify JDK file " \
- "location in local filesystem using --jdk-location command " \
- "line argument.".format("{0}", jdk_url)
- try:
- force_download_file(jdk_url, dest_file)
- print 'Successfully downloaded JDK distribution to ' + dest_file
- except FatalException:
- raise
- except Exception, e:
- err = jdk_download_fail_msg.format(str(e))
- raise FatalException(1, err)
+class AmbariUserChecks(object):
+ def __init__(self):
+ self.NR_USER_CHANGE_PROMPT = ""
+ self.NR_USER_CUSTOMIZE_PROMPT = ""
+ self.NR_DEFAULT_USER = ""
+ self.NR_USER_COMMENT = "Ambari user"
+ def do_checks(self):
+ try:
+ user = read_ambari_user()
+ create_user = False
+ update_user_setting = False
+ if user is not None:
+ create_user = get_YN_input(self.NR_USER_CHANGE_PROMPT.format(user), False)
+ update_user_setting = create_user # Only if we will create another user
+ else: # user is not configured yet
+ update_user_setting = True # Write configuration anyway
+ create_user = get_YN_input(self.NR_USER_CUSTOMIZE_PROMPT, False)
+ if not create_user:
+ user = self.NR_DEFAULT_USER
+
+ if create_user:
+ (retcode, user) = self._create_custom_user()
+ if retcode != 0:
+ return retcode
+
+ if update_user_setting:
+ write_property(NR_USER_PROPERTY, user)
+
+ adjust_directory_permissions(user)
+ except OSError as e:
+ print_error_msg("Failed: %s" % str(e))
+ return 4
+ except Exception as e:
+ print_error_msg("Unexpected error %s" % str(e))
+ return 1
+ return 0
-def download_and_install_jdk(args):
- properties = get_ambari_properties()
- if properties == -1:
- err = "Error getting ambari properties"
- raise FatalException(-1, err)
+ def _create_custom_user(self):
+ pass
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class AmbariUserChecksWindows(AmbariUserChecks):
+ def __init__(self):
+ super(AmbariUserChecksWindows, self).__init__()
+
+ self.NR_USER_CHANGE_PROMPT = "Ambari-server service is configured to run under user '{0}'. Change this setting [y/n] (n)? "
+ self.NR_USER_CUSTOMIZE_PROMPT = "Customize user account for ambari-server service [y/n] (n)? "
+ self.NR_DEFAULT_USER = "NT AUTHORITY\SYSTEM"
+
+ def _create_custom_user(self):
+ user = get_validated_string_input(
+ "Enter user account for ambari-server service ({0}):".format(self.NR_DEFAULT_USER),
+ self.NR_DEFAULT_USER, None,
+ "Invalid username.",
+ False
+ )
+ if user == self.NR_DEFAULT_USER:
+ return 0, user
+ password = get_validated_string_input("Enter password for user {0}:".format(user), "", None, "Password", True, False)
+
+ from ambari_commons.os_windows import UserHelper
+
+ uh = UserHelper()
+
+ status, message = uh.create_user(user,password)
+ if status == UserHelper.USER_EXISTS:
+ print_info_msg("User {0} already exists, make sure that you typed correct password for user, "
+ "skipping user creation".format(user))
+
+ elif status == UserHelper.ACTION_FAILED: # fail
+ print_warning_msg("Can't create user {0}. Failed with message {1}".format(user, message))
+ return UserHelper.ACTION_FAILED, None
+
+ # setting SeServiceLogonRight to user
+
+ status, message = uh.add_user_privilege(user, 'SeServiceLogonRight')
+ if status == UserHelper.ACTION_FAILED:
+ print_warning_msg("Can't add SeServiceLogonRight to user {0}. Failed with message {1}".format(user, message))
+ return UserHelper.ACTION_FAILED, None
+
+ print_info_msg("User configuration is done.")
+ print_warning_msg("When using non SYSTEM user make sure that your user have read\write access to log directories and "
+ "all server directories. In case of integrated authentication for SQL Server make sure that your "
+ "user properly configured to use ambari and metric database.")
+ #storing username and password in os.environ temporary to pass them to service
+ os.environ[SERVICE_USERNAME_KEY] = user
+ os.environ[SERVICE_PASSWORD_KEY] = password
+ return 0, user
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class AmbariUserChecksLinux(AmbariUserChecks):
+ def __init__(self):
+ super(AmbariUserChecksLinux, self).__init__()
+
+ self.NR_USER_CHANGE_PROMPT = "Ambari-server daemon is configured to run under user '{0}'. Change this setting [y/n] (n)? "
+ self.NR_USER_CUSTOMIZE_PROMPT = "Customize user account for ambari-server daemon [y/n] (n)? "
+ self.NR_DEFAULT_USER = "root"
+
+ self.NR_USERADD_CMD = 'useradd -M --comment "{1}" ' \
+ '--shell %s -d /var/lib/ambari-server/keys/ {0}' % locate_file('nologin', '/sbin')
+
+ def _create_custom_user(self):
+ user = get_validated_string_input(
+ "Enter user account for ambari-server daemon (root):",
+ "root",
+ "^[a-z_][a-z0-9_-]{1,31}$",
+ "Invalid username.",
+ False
+ )
+
+ print_info_msg("Trying to create user {0}".format(user))
+ command = self.NR_USERADD_CMD.format(user, self.NR_USER_COMMENT)
+ retcode, out, err = run_os_command(command)
+ if retcode == 9: # 9 = username already in use
+ print_info_msg("User {0} already exists, "
+ "skipping user creation".format(user))
+
+ elif retcode != 0: # fail
+ print_warning_msg("Can't create user {0}. Command {1} "
+ "finished with {2}: \n{3}".format(user, command, retcode, err))
+ return retcode, None
+
+ print_info_msg("User configuration is done.")
+ return 0, user
- conf_file = properties.fileName
- ok = False
- jcePolicyWarn = "JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos," \
- "please make sure JCE Unlimited Strength Jurisdiction Policy Files are valid on all hosts."
+def check_ambari_user():
+ return AmbariUserChecks().do_checks()
+
+
+# ## Firewall ###
+
+def check_firewall():
+ firewall_obj = Firewall().getFirewallObject()
+ print firewall_obj.MESSAGE_CHECK_FIREWALL
+ firewall_on = firewall_obj.check_iptables()
+ if firewall_obj.stderrdata and len(firewall_obj.stderrdata) > 0:
+ print firewall_obj.stderrdata
+ if firewall_on:
+ print_warning_msg("%s is running. Confirm the necessary Ambari ports are accessible. " %
+ firewall_obj.FIREWALL_SERVICE_NAME +
+ "Refer to the Ambari documentation for more details on ports.")
+ ok = get_YN_input("OK to continue [y/n] (y)? ", True)
+ if not ok:
+ raise FatalException(1, None)
- if args.java_home:
- if not os.path.exists(args.java_home) or not os.path.isfile(os.path.join(args.java_home, JAVA_EXE_SUBPATH)):
- err = "Path to java home " + args.java_home + " or java binary file does not exists"
- raise FatalException(1, err)
- print_warning_msg("JAVA_HOME " + args.java_home + " must be valid on ALL hosts")
- print_warning_msg(jcePolicyWarn)
+# ## JDK ###
- properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
- properties.removeOldProp(JDK_NAME_PROPERTY)
- properties.removeOldProp(JCE_NAME_PROPERTY)
- update_properties(properties)
+class JDKSetup(object):
+ def __init__(self):
+ self.JDK_DEFAULT_CONFIGS = []
+
+ self.JDK_PROMPT = "[{0}] {1}\n"
+ self.JDK_CUSTOM_CHOICE_PROMPT = "[{0}] - Custom JDK\n==============================================================================\nEnter choice ({1}): "
+ self.JDK_VALID_CHOICES = "^[{0}{1:d}]$"
+ self.JDK_MIN_FILESIZE = 5000
+ self.JAVA_BIN = ""
+
+ self.jdk_index = 0
+
+ #
+ # Downloads and installs the JDK and the JCE policy archive
+ #
+ def download_and_install_jdk(self, args):
+ properties = get_ambari_properties()
+ if properties == -1:
+ err = "Error getting ambari properties"
+ raise FatalException(-1, err)
+
+ conf_file = properties.fileName
+ ok = False
+ jcePolicyWarn = "JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos," \
+ "please make sure JCE Unlimited Strength Jurisdiction Policy Files are valid on all hosts."
+ jdk_num = "1"
+
+ if args.java_home:
+ if not os.path.exists(args.java_home) or not os.path.isfile(os.path.join(args.java_home, configDefaults.JAVA_EXE_SUBPATH)):
+ err = "Path to java home " + args.java_home + " or java binary file does not exists"
+ raise FatalException(1, err)
- os_ensure_java_home_env_var_is_set(args.java_home)
- return 0
- else:
- global JDK_INDEX
+ print_warning_msg("JAVA_HOME " + args.java_home + " must be valid on ALL hosts")
+ print_warning_msg(jcePolicyWarn)
+
+ properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
+ properties.removeOldProp(JDK_NAME_PROPERTY)
+ properties.removeOldProp(JCE_NAME_PROPERTY)
+ update_properties(properties)
+
+ self._ensure_java_home_env_var_is_set(args.java_home)
+ return 0
+ else:
+ java_home_var = get_JAVA_HOME()
- java_home_var = get_JAVA_HOME()
- if java_home_var:
if args.silent:
- change_jdk = False
- else:
- change_jdk = get_YN_input("Do you want to change Oracle JDK [y/n] (n)? ", False)
- if not change_jdk:
- os_ensure_java_home_env_var_is_set(java_home_var)
- return 0
- #Handle silent JDK setup when args.silent is set
- elif args.silent:
- #No java_home_var set, detect if java is already installed
- if os.environ.has_key(JAVA_HOME):
- args.java_home = os.environ[JAVA_HOME]
-
- properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
- properties.removeOldProp(JDK_NAME_PROPERTY)
- properties.removeOldProp(JCE_NAME_PROPERTY)
- update_properties(properties)
-
- os_ensure_java_home_env_var_is_set(args.java_home)
- return 0
- else:
+ if not java_home_var:
+ #No java_home_var set, detect if java is already installed
+ if os.environ.has_key(JAVA_HOME):
+ args.java_home = os.environ[JAVA_HOME]
+
+ properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
+ properties.removeOldProp(JDK_NAME_PROPERTY)
+ properties.removeOldProp(JCE_NAME_PROPERTY)
+ update_properties(properties)
+
+ self._ensure_java_home_env_var_is_set(args.java_home)
+ return 0
+ else:
+ # For now, changing the existing JDK to make sure we use a supported one
+ pass
+
#Continue with the normal setup, taking the first listed JDK version as the default option
jdk_num = "1"
- (jdks, jdk_choice_prompt, jdk_valid_choices, custom_jdk_number) = populate_jdk_configs(properties, jdk_num)
- else:
- jdk_num = str(JDK_INDEX + 1)
- (jdks, jdk_choice_prompt, jdk_valid_choices, custom_jdk_number) = populate_jdk_configs(properties, jdk_num)
-
- jdk_num = get_validated_string_input(
- jdk_choice_prompt,
- jdk_num,
- jdk_valid_choices,
- "Invalid number.",
- False
- )
-
- java_bin = "java"
- if OSCheck.is_windows_family():
- java_bin = "java.exe"
-
- if jdk_num == str(custom_jdk_number):
- print_warning_msg("JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts.")
- print_warning_msg(jcePolicyWarn)
- args.java_home = get_validated_string_input("Path to JAVA_HOME: ", None, None, None, False, False)
- if not os.path.exists(args.java_home) or not os.path.isfile(os.path.join(args.java_home, "bin", java_bin)):
- err = "Java home path or java binary file is unavailable. Please put correct path to java home."
- raise FatalException(1, err)
- print "Validating JDK on Ambari Server...done."
+ (jdks, jdk_choice_prompt, jdk_valid_choices, custom_jdk_number) = self._populate_jdk_configs(properties, jdk_num)
+ else:
+ if java_home_var:
+ change_jdk = get_YN_input("Do you want to change Oracle JDK [y/n] (n)? ", False)
+ if not change_jdk:
+ self._ensure_java_home_env_var_is_set(java_home_var)
+ return 0
+
+ #Continue with the normal setup
+ jdk_num = str(self.jdk_index + 1)
+ (jdks, jdk_choice_prompt, jdk_valid_choices, custom_jdk_number) = self._populate_jdk_configs(properties, jdk_num)
+
+ jdk_num = get_validated_string_input(
+ jdk_choice_prompt,
+ jdk_num,
+ jdk_valid_choices,
+ "Invalid number.",
+ False
+ )
+
+ if jdk_num == str(custom_jdk_number):
+ print_warning_msg("JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts.")
+ print_warning_msg(jcePolicyWarn)
+ args.java_home = get_validated_string_input("Path to JAVA_HOME: ", None, None, None, False, False)
+ if not os.path.exists(args.java_home) or not os.path.isfile(os.path.join(args.java_home, "bin", self.JAVA_BIN)):
+ err = "Java home path or java binary file is unavailable. Please put correct path to java home."
+ raise FatalException(1, err)
+ print "Validating JDK on Ambari Server...done."
- properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
- properties.removeOldProp(JDK_NAME_PROPERTY)
- properties.removeOldProp(JCE_NAME_PROPERTY)
- update_properties(properties)
+ properties.process_pair(JAVA_HOME_PROPERTY, args.java_home)
+ properties.removeOldProp(JDK_NAME_PROPERTY)
+ properties.removeOldProp(JCE_NAME_PROPERTY)
+ update_properties(properties)
- os_ensure_java_home_env_var_is_set(args.java_home)
- return 0
+ self._ensure_java_home_env_var_is_set(args.java_home)
+ return 0
- JDK_INDEX = int(jdk_num) - 1
- jdk_cfg = jdks[JDK_INDEX]
+ self.jdk_index = int(jdk_num) - 1
+ jdk_cfg = jdks[self.jdk_index]
- try:
- resources_dir = properties[RESOURCES_DIR_PROPERTY]
- except (KeyError), e:
- err = 'Property ' + str(e) + ' is not defined at ' + conf_file
- raise FatalException(1, err)
+ try:
+ resources_dir = properties[RESOURCES_DIR_PROPERTY]
+ except (KeyError), e:
+ err = 'Property ' + str(e) + ' is not defined at ' + conf_file
+ raise FatalException(1, err)
- dest_file = os.path.abspath(os.path.join(resources_dir, jdk_cfg.dest_file))
- if os.path.exists(dest_file):
- print "JDK already exists, using " + dest_file
- else:
- if args.silent:
- print "Accepting the JDK license terms by default..."
+ dest_file = os.path.abspath(os.path.join(resources_dir, jdk_cfg.dest_file))
+ if os.path.exists(dest_file):
+ print "JDK already exists, using " + dest_file
else:
- ok = get_YN_input("To download the Oracle JDK you must accept the "
- "license terms found at "
- "http://www.oracle.com/technetwork/java/javase/"
- "terms/license/index.html and not accepting will "
- "cancel the Ambari Server setup.\nDo you accept the "
- "Oracle Binary Code License Agreement [y/n] (y)? ", True)
- if not ok:
- print 'Exiting...'
- sys.exit(1)
-
- jdk_url = jdk_cfg.url
-
- print 'Downloading JDK from ' + jdk_url + ' to ' + dest_file
- _dowload_jdk(jdk_url, dest_file)
-
- try:
- (retcode, out) = install_jdk(dest_file, jdk_cfg.inst_dir)
- except Exception, e:
- print "Installation of JDK has failed: %s\n" % e.message
- file_exists = os.path.isfile(dest_file)
- if file_exists:
if args.silent:
- ok = False
- else:
- ok = get_YN_input("JDK found at " + dest_file + ". "
- "Would you like to re-download the JDK [y/n] (y)? ", True)
- if not ok:
- err = "Unable to install JDK. Please remove JDK file found at " + \
- dest_file + " and re-run Ambari Server setup"
- raise FatalException(1, err)
+ print "Accepting the JDK license terms by default..."
else:
- jdk_url = jdk_cfg.url
-
- print 'Re-downloading JDK from ' + jdk_url + ' to ' + dest_file
- _dowload_jdk(jdk_url, dest_file)
- print 'Successfully re-downloaded JDK distribution to ' + dest_file
-
- try:
- (retcode, out) = install_jdk(dest_file, jdk_cfg.inst_dir)
- except Exception, e:
- print "Installation of JDK was failed: %s\n" % e.message
- err = "Unable to install JDK. Please remove JDK, file found at " + \
+ ok = get_YN_input("To download the Oracle JDK you must accept the "
+ "license terms found at "
+ "http://www.oracle.com/technetwork/java/javase/"
+ "terms/license/index.html and not accepting will "
+ "cancel the Ambari Server setup.\nDo you accept the "
+ "Oracle Binary Code License Agreement [y/n] (y)? ", True)
+ if not ok:
+ print 'Exiting...'
+ sys.exit(1)
+
+ jdk_url = jdk_cfg.url
+
+ print 'Downloading JDK from ' + jdk_url + ' to ' + dest_file
+ self._download_jdk(jdk_url, dest_file)
+
+ try:
+ (retcode, out, java_home_dir) = self._install_jdk(dest_file, jdk_cfg)
+ except Exception, e:
+ print "Installation of JDK has failed: %s\n" % str(e)
+ file_exists = os.path.isfile(dest_file)
+ if file_exists:
+ if args.silent:
+ ok = False
+ else:
+ ok = get_YN_input("JDK found at " + dest_file + ". "
+ "Would you like to re-download the JDK [y/n] (y)? ", True)
+ if not ok:
+ err = "Unable to install JDK. Please remove JDK file found at " + \
dest_file + " and re-run Ambari Server setup"
raise FatalException(1, err)
+ else:
+ jdk_url = jdk_cfg.url
- else:
- err = "Unable to install JDK. File " + dest_file + " does not exist, " \
- "please re-run Ambari Server setup"
- raise FatalException(1, err)
+ print 'Re-downloading JDK from ' + jdk_url + ' to ' + dest_file
+ self._download_jdk(jdk_url, dest_file)
+ print 'Successfully re-downloaded JDK distribution to ' + dest_file
- properties.process_pair(JDK_NAME_PROPERTY, jdk_cfg.dest_file)
- properties.process_pair(JAVA_HOME_PROPERTY, jdk_cfg.inst_dir)
+ try:
+ (retcode, out) = self._install_jdk(dest_file, jdk_cfg)
+ except Exception, e:
+ print "Installation of JDK was failed: %s\n" % str(e)
+ err = "Unable to install JDK. Please remove JDK, file found at " + \
+ dest_file + " and re-run Ambari Server setup"
+ raise FatalException(1, err)
- try:
- download_jce_policy(jdk_cfg, resources_dir, properties)
- except FatalException, e:
- print "JCE Policy files are required for secure HDP setup. Please ensure " \
- " all hosts have the JCE unlimited strength policy 6, files."
- print_error_msg("Failed to download JCE policy files:")
- if e.reason is not None:
- print_error_msg("\nREASON: {0}".format(e.reason))
- # TODO: We don't fail installation if download_jce_policy fails. Is it OK?
+ else:
+ err = "Unable to install JDK. File " + dest_file + " does not exist, " \
+ "please re-run Ambari Server setup"
+ raise FatalException(1, err)
- update_properties(properties)
+ properties.process_pair(JDK_NAME_PROPERTY, jdk_cfg.dest_file)
+ properties.process_pair(JAVA_HOME_PROPERTY, java_home_dir)
- os_ensure_java_home_env_var_is_set(jdk_cfg.inst_dir)
+ try:
+ self._download_jce_policy(jdk_cfg, resources_dir, properties)
+ except FatalException, e:
+ print "JCE Policy files are required for secure HDP setup. Please ensure " \
+ " all hosts have the JCE unlimited strength policy 6, files."
+ print_error_msg("Failed to download JCE policy files:")
+ if e.reason is not None:
+ print_error_msg("\nREASON: {0}".format(e.reason))
+ # TODO: We don't fail installation if _download_jce_policy fails. Is it OK?
- return 0
+ update_properties(properties)
+ self._ensure_java_home_env_var_is_set(java_home_dir)
-def download_jce_policy(jdk_cfg, resources_dir, properties):
- jcpol_url = jdk_cfg.jcpol_url
- dest_file = os.path.abspath(os.path.join(resources_dir, jdk_cfg.dest_jcpol_file))
+ return 0
- if not os.path.exists(dest_file):
- print 'Downloading JCE Policy archive from ' + jcpol_url + ' to ' + dest_file
+ def _populate_jdk_configs(self, properties, jdk_num):
+ if properties.has_key(JDK_RELEASES):
+ jdk_names = properties[JDK_RELEASES].split(',')
+ jdks = []
+ for jdk_name in jdk_names:
+ jdkR = JDKRelease.from_properties(properties, jdk_name)
+ jdks.append(jdkR)
+ else:
+ jdks = self.JDK_DEFAULT_CONFIGS
+
+ n_config = 1
+ jdk_choice_prompt = ''
+ jdk_choices = ''
+ for jdk in jdks:
+ jdk_choice_prompt += self.JDK_PROMPT.format(n_config, jdk.desc)
+ jdk_choices += str(n_config)
+ n_config += 1
+
+ jdk_choice_prompt += self.JDK_CUSTOM_CHOICE_PROMPT.format(n_config, jdk_num)
+ jdk_valid_choices = self.JDK_VALID_CHOICES.format(jdk_choices, n_config)
+
+ return (jdks, jdk_choice_prompt, jdk_valid_choices, n_config)
+
+ def _download_jdk(self, jdk_url, dest_file):
+ jdk_download_fail_msg = " Failed to download JDK: {0}. Please check that the " \
+ "JDK is available at {1}. Also you may specify JDK file " \
+ "location in local filesystem using --jdk-location command " \
+ "line argument.".format("{0}", jdk_url)
try:
- force_download_file(jcpol_url, dest_file)
+ force_download_file(jdk_url, dest_file)
- print 'Successfully downloaded JCE Policy archive to ' + dest_file
- properties.process_pair(JCE_NAME_PROPERTY, jdk_cfg.dest_jcpol_file)
+ print 'Successfully downloaded JDK distribution to ' + dest_file
except FatalException:
raise
except Exception, e:
- err = 'Failed to download JCE Policy archive: ' + str(e)
+ err = jdk_download_fail_msg.format(str(e))
+ raise FatalException(1, err)
+
+ def _download_jce_policy(self, jdk_cfg, resources_dir, properties):
+ jcpol_url = jdk_cfg.jcpol_url
+ dest_file = os.path.abspath(os.path.join(resources_dir, jdk_cfg.dest_jcpol_file))
+
+ if not os.path.exists(dest_file):
+ print 'Downloading JCE Policy archive from ' + jcpol_url + ' to ' + dest_file
+ try:
+ force_download_file(jcpol_url, dest_file)
+
+ print 'Successfully downloaded JCE Policy archive to ' + dest_file
+ properties.process_pair(JCE_NAME_PROPERTY, jdk_cfg.dest_jcpol_file)
+ except FatalException:
+ raise
+ except Exception, e:
+ err = 'Failed to download JCE Policy archive: ' + str(e)
+ raise FatalException(1, err)
+ else:
+ print "JCE Policy archive already exists, using " + dest_file
+
+ # Base implementation, overriden in the subclasses
+ def _install_jdk(self, java_inst_file, java_home_dir):
+ pass
+
+ # Base implementation, overriden in the subclasses
+ def _ensure_java_home_env_var_is_set(self, java_home_dir):
+ pass
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class JDKSetupWindows(JDKSetup):
+ def __init__(self):
+ super(JDKSetupWindows, self).__init__()
+ self.JDK_DEFAULT_CONFIGS = [
+ JDKRelease("jdk7.67", "Oracle JDK 1.7.67",
+ "http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe", "jdk-7u67-windows-x64.exe",
+ "http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip", "UnlimitedJCEPolicyJDK7.zip",
+ "C:\\jdk1.7.0_67",
+ "Creating (jdk.*)/jre")
+ ]
+
+ self.JAVA_BIN = "java.exe"
+
+ def _install_jdk(self, java_inst_file, jdk_cfg):
+ jdk_inst_dir = jdk_cfg.inst_dir
+ print "Installing JDK to {0}".format(jdk_inst_dir)
+
+ if not os.path.exists(jdk_inst_dir):
+ os.makedirs(jdk_inst_dir)
+
+ if java_inst_file.endswith(".exe"):
+ (dirname, filename) = os.path.split(java_inst_file)
+ installLogFilePath = os.path.join(configDefaults.OUT_DIR, filename + "-install.log")
+ #jre7u67.exe /s INSTALLDIR=<dir> STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-server\\jre7u67.exe-install.log
+ installCmd = [
+ java_inst_file,
+ "/s",
+ "INSTALLDIR=" + jdk_inst_dir,
+ "STATIC=1",
+ "WEB_JAVA=0",
+ "/L",
+ installLogFilePath
+ ]
+ retcode, out, err = run_os_command(installCmd)
+ #TODO: support .msi file installations
+ #msiexec.exe jre.msi /s INSTALLDIR=<dir> STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-server\\jre7u67-install.log ?
+ else:
+ err = "JDK installation failed.Unknown file mask."
raise FatalException(1, err)
- else:
- print "JCE Policy archive already exists, using " + dest_file
+ if retcode == 1603:
+ # JDK already installed
+ print "JDK already installed in {0}".format(jdk_inst_dir)
+ retcode = 0
+ else:
+ if retcode != 0:
+ err = "Installation of JDK returned exit code %s" % retcode
+ raise FatalException(retcode, err)
+ print "Successfully installed JDK to {0}".format(jdk_inst_dir)
-def install_jdk(java_inst_file, java_home_dir):
- return os_install_jdk(java_inst_file, java_home_dir)
+ # Don't forget to adjust the JAVA_HOME env var
+ return (retcode, out, jdk_inst_dir)
+
+ def _ensure_java_home_env_var_is_set(self, java_home_dir):
+ if not os.environ.has_key(JAVA_HOME) or os.environ[JAVA_HOME] != java_home_dir:
+ java_home_dir_unesc = compress_backslashes(java_home_dir)
+ retcode, out, err = run_os_command("SETX {0} {1} /M".format(JAVA_HOME, java_home_dir_unesc))
+ if retcode != 0:
+ print_warning_msg("SETX output: " + out)
+ print_warning_msg("SETX error output: " + err)
+ err = "Setting JAVA_HOME failed. Exit code={0}".format(retcode)
+ raise FatalException(1, err)
+
+ os.environ[JAVA_HOME] = java_home_dir
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class JDKSetupLinux(JDKSetup):
+ def __init__(self):
+ super(JDKSetupLinux, self).__init__()
+ self.JDK_DEFAULT_CONFIGS = [
+ JDKRelease("jdk6.31", "Oracle JDK 1.6",
+ "http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-6u31-linux-x64.bin", "jdk-6u31-linux-x64.bin",
+ "http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip", "jce_policy-6.zip",
+ "/usr/jdk64/jdk1.6.0_31",
+ "Creating (jdk.*)/jre")
+ ]
+
+ self.JAVA_BIN = "java"
+
+ self.CREATE_JDK_DIR_CMD = "/bin/mkdir -p {0}"
+ self.MAKE_FILE_EXECUTABLE_CMD = "chmod a+x {0}"
+
+ # use --no-same-owner when running as root to prevent uucp as the user (AMBARI-6478)
+ self.UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}"
+
+ def _install_jdk(self, java_inst_file, jdk_cfg):
+ jdk_inst_dir = jdk_cfg.inst_dir
+ print "Installing JDK to {0}".format(jdk_inst_dir)
+
+ retcode, out, err = run_os_command(self.CREATE_JDK_DIR_CMD.format(jdk_inst_dir))
+ savedPath = os.getcwd()
+ os.chdir(jdk_inst_dir)
+
+ try:
+ if java_inst_file.endswith(".bin"):
+ retcode, out, err = run_os_command(self.MAKE_FILE_EXECUTABLE_CMD.format(java_inst_file))
+ retcode, out, err = run_os_command(java_inst_file + ' -noregister')
+ elif java_inst_file.endswith(".gz"):
+ retcode, out, err = run_os_command(self.UNTAR_JDK_ARCHIVE.format(java_inst_file))
+ else:
+ err = "JDK installation failed.Unknown file mask."
+ raise FatalException(1, err)
+ finally:
+ os.chdir(savedPath)
+
+ if retcode != 0:
+ err = "Installation of JDK returned exit code %s" % retcode
+ raise FatalException(retcode, err)
+
+ jdk_version = re.search(jdk_cfg.reg_exp, out).group(1)
+ java_home_dir = os.path.join(jdk_inst_dir, jdk_version)
+
+ print "Successfully installed JDK to {0}".format(jdk_inst_dir)
+ return (retcode, out, java_home_dir)
+
+ def _ensure_java_home_env_var_is_set(self, java_home_dir):
+ #No way to do this in Linux. Best we can is to set the process environment variable.
+ os.environ[JAVA_HOME] = java_home_dir
+
+def download_and_install_jdk(options):
+ jdk_setup_obj = JDKSetup()
+ return jdk_setup_obj.download_and_install_jdk(options)
+
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def get_master_os_type():
+ return OS_TYPE + OS_VERSION
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def get_master_os_type():
+ return OS_FAMILY + OS_VERSION
#
# Configures the OS settings in ambari properties.
@@ -329,11 +699,12 @@ def configure_os_settings():
except (KeyError):
print_error_msg("os_type is not set in the properties file. Setting it now.")
- if OSCheck.is_windows_family():
- master_os_type = OS_TYPE + OS_VERSION
- else:
- # MacOS not supported
- master_os_type = OS_FAMILY + OS_VERSION
+ os_system = platform.system()
+ if os_system != 'Linux' and os_system != 'Windows':
+ print_error_msg("Only Linux and Windows systems are supported")
+ return -1
+
+ master_os_type = get_master_os_type()
write_property(OS_TYPE_PROPERTY, master_os_type)
return 0
@@ -343,6 +714,9 @@ def configure_os_settings():
# JDBC
#
+def _check_jdbc_options(options):
+ return (options.jdbc_driver is not None and options.jdbc_db is not None)
+
def proceedJDBCProperties(args):
if not os.path.isfile(args.jdbc_driver):
err = "File {0} does not exist!".format(args.jdbc_driver)
@@ -352,6 +726,17 @@ def proceedJDBCProperties(args):
err = "Unsupported database name {0}. Please see help for more information.".format(args.jdbc_db)
raise FatalException(1, err)
+ _cache_jdbc_driver(args)
+
+# No JDBC driver caching in Windows at this point. Will cache it along with the integrated authentication dll into a
+# zip archive at a later moment.
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def _cache_jdbc_driver(args):
+ pass
+
+#TODO JDBC driver caching duplicates the LinuxDBMSConfig._install_jdbc_driver() functionality
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def _cache_jdbc_driver(args):
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
@@ -376,31 +761,176 @@ def proceedJDBCProperties(args):
shutil.copy(args.jdbc_driver, resources_dir)
except Exception, e:
err = "Can not copy file {0} to {1} due to: {2} . Please check file " \
- "permissions and free disk space.".format(args.jdbc_driver, resources_dir, e)
+ "permissions and free disk space.".format(args.jdbc_driver, resources_dir, str(e))
raise FatalException(1, err)
os.symlink(os.path.join(resources_dir, jdbc_name), jdbc_symlink)
print "JDBC driver was successfully initialized."
def check_jdbc_drivers(args):
- os_setup_jdbc_drivers(args)
- pass
+ properties = get_ambari_properties()
+ if properties == -1:
+ print_error_msg("Error getting ambari properties")
+ return -1
+
+ factory = DBMSConfigFactory()
+
+ #Only support SQL Server
+ dbms = factory.create(args, properties)
+ if dbms.ensure_jdbc_driver_installed(properties):
+ # Now save the properties file
+ return update_properties(properties)
-# Ask user for database conenction properties
-def prompt_db_properties(args):
- if not args.silent:
- def_option = 'y' if args.must_set_database_options else 'n'
- ok = get_YN_input("Enter advanced database configuration [y/n] ({})? ".format(def_option), args.must_set_database_options)
+
+# Ask user for database connection properties
+def prompt_db_properties(options):
+ if not options.silent:
+ def_option = 'y' if options.must_set_database_options else 'n'
+ ok = get_YN_input("Enter advanced database configuration [y/n] ({0})?".format(def_option), options.must_set_database_options)
if not ok:
+ #TODO Display the database options
return False
print 'Configuring database...'
- #TODO: Add here code for DBMS selection, in case we want to support other databases besides SQL Server
+ factory = DBMSConfigFactory()
+
+ options.dbms_index = factory.select_dbms(options)
return True
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def _setup_database(options):
+ properties = get_ambari_properties()
+ if properties == -1:
+ raise FatalException(-1, "Error getting ambari properties")
+
+ #Ensure the default database host is set
+ options.default_database_host = "localhost\\SQLEXPRESS"
+
+ factory = DBMSConfigFactory()
+
+ #Only support SQL Server
+ dbmsAmbari = factory.create(options, properties, "Ambari")
+ resultA = dbmsAmbari.configure_database(options, properties)
+
+ #By default, use the same server for Metrics
+ options.default_database_host = dbmsAmbari.database_host
+
+ dbmsMetrics = factory.create(options, properties, "Metrics")
+ resultM = dbmsMetrics.configure_database(options, properties)
+
+ # Now save the properties file
+ if resultA or resultM:
+ update_properties(properties)
+
+ dbmsAmbari.setup_database()
+ dbmsMetrics.setup_database()
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def _setup_database(options):
+ properties = get_ambari_properties()
+ if properties == -1:
+ raise FatalException(-1, "Error getting ambari properties")
+
+ factory = DBMSConfigFactory()
+
+ dbmsAmbari = factory.create(options, properties, "Ambari")
+ resultA = dbmsAmbari.configure_database(options, properties)
+
+ # Now save the properties file
+ if resultA:
+ update_properties(properties)
+
+ dbmsAmbari.setup_database()
+
+def _createDefDbFactory(options):
+ properties = get_ambari_properties()
+ if properties == -1:
+ raise FatalException(-1, "Error getting ambari properties")
+ if not (properties.getPropertyDict().has_key(JDBC_URL_PROPERTY) and
+ properties.getPropertyDict().has_key(JDBC_RCA_URL_PROPERTY) and
+ properties.getPropertyDict().has_key(JDBC_METRICS_URL_PROPERTY)):
+ raise FatalException(-1, "Ambari Server not set up yet. Nothing to reset.")
+
+ empty_options = optparse.Values()
+ empty_options.silent = options.silent
+ empty_options.dbms_index = options.dbms_index
+ empty_options.database_host = ""
+ empty_options.database_port = ""
+ empty_options.database_name = ""
+ empty_options.database_windows_auth = False
+ empty_options.database_username = ""
+ empty_options.database_password = ""
+ empty_options.init_db_script_file = ""
+ empty_options.cleanup_db_script_file = ""
+ empty_options.init_metrics_db_script_file = ""
+ empty_options.cleanup_metrics_db_script_file = ""
+
+ factory = DBMSConfigFactory()
+
+ return empty_options, factory, properties
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def _reset_database(options):
+ empty_options, factory, properties = _createDefDbFactory(options)
+
+ dbmsAmbari = factory.create(empty_options, properties, "Ambari")
+ dbmsAmbari.reset_database()
+
+ dbmsMetrics = factory.create(empty_options, properties, "Metrics")
+ dbmsMetrics.reset_database()
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def _reset_database(options):
+ #TODO Ensure the correct DBMS is constructed, with the correct server, port, user, pwd
+ empty_options, factory, properties = _createDefDbFactory(options)
+
+ dbmsAmbari = factory.create(empty_options, properties)
+ dbmsAmbari.reset_database()
+
+#
+# Extract the system views
+#
+def extract_views():
+ jdk_path = find_jdk()
+ if jdk_path is None:
+ print_error_msg("No JDK found, please run the \"setup\" "
+ "command to install a JDK automatically or install any "
+ "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
+ return 1
+
+ properties = get_ambari_properties()
+ if properties == -1:
+ print_error_msg("Error getting ambari properties")
+ return -1
+
+ if not VIEWS_DIR_PROPERTY in properties.keys():
+ vdir = configDefaults.DEFAULT_VIEWS_DIR
+ else:
+ vdir = properties.get_property(VIEWS_DIR_PROPERTY)
+
+ files = [f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir,f))]
+ for f in files:
+
+ command = VIEW_EXTRACT_CMD.format(jdk_path, get_full_ambari_classpath(), os.path.join(vdir,f))
+
+ retcode, stdout, stderr = run_os_command(command)
+ if retcode == 0:
+ sys.stdout.write(f + "\n")
+ elif retcode == 2:
+ sys.stdout.write("Error extracting " + f + "\n")
+ else:
+ sys.stdout.write(".")
+ sys.stdout.flush()
+
+ print_info_msg("Return code from extraction of view archive " + f + ": " +
+ str(retcode))
+
+ sys.stdout.write("\n")
+ return 0
+
#
# Setup the Ambari Server.
@@ -416,7 +946,7 @@ def setup(options):
raise FatalException(4, err)
# proceed jdbc properties if they were set
- if os_check_jdbc_options(options):
+ if _check_jdbc_options(options):
proceedJDBCProperties(options)
return
@@ -430,18 +960,17 @@ def setup(options):
err = 'Failed to create user. Exiting.'
raise FatalException(retcode, err)
- print MESSAGE_CHECK_FIREWALL
- os_check_firewall()
+ check_firewall()
# proceed jdbc properties if they were set
- if os_check_jdbc_options(options):
+ if _check_jdbc_options(options):
proceedJDBCProperties(options)
print 'Checking JDK...'
try:
download_and_install_jdk(options)
except FatalException as e:
- err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
+ err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(str(e))
raise FatalException(e.code, err)
print 'Completing setup...'
@@ -452,10 +981,18 @@ def setup(options):
if prompt_db_properties(options):
#DB setup should be done last after doing any setup.
- os_setup_database(options)
+ _setup_database(options)
check_jdbc_drivers(options)
- pass
+
+ print 'Extracting system views...'
+ retcode = extract_views()
+ if not retcode == 0:
+ err = 'Error while extracting system views. Exiting'
+ raise FatalException(retcode, err)
+
+ # we've already done this, but new files were created so run it one time.
+ adjust_directory_permissions(read_ambari_user())
#
# Upgrades the Ambari Server.
@@ -520,11 +1057,14 @@ def reset(options, serviceClass):
err = "Ambari Server 'reset' cancelled"
raise FatalException(1, err)
- os_reset_database(options)
+ _reset_database(options)
pass
def is_server_running(serviceClass):
+ from ambari_commons.os_windows import SERVICE_STATUS_STARTING, SERVICE_STATUS_RUNNING, SERVICE_STATUS_STOPPING, \
+ SERVICE_STATUS_STOPPED, SERVICE_STATUS_NOT_INSTALLED
+
statusStr = serviceClass.QueryStatus()
if statusStr in(SERVICE_STATUS_STARTING, SERVICE_STATUS_RUNNING, SERVICE_STATUS_STOPPING):
return True, ""
[6/7] ambari git commit: AMBARI-8373 Refactor the OS-dependent Ambari
Server Windows components
Posted by fb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari-server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py
index 89caa2e..f360f74 100755
--- a/ambari-server/src/main/python/ambari-server.py
+++ b/ambari-server/src/main/python/ambari-server.py
@@ -39,7 +39,11 @@ import datetime
import tempfile
import random
import pwd
+from ambari_commons.exceptions import NonFatalException, FatalException
+from ambari_commons.os_utils import is_root
+from ambari_server.properties import Properties
from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException
+from ambari_server.serverSetup import setup
import json
import base64
from threading import Thread
@@ -97,11 +101,6 @@ ambari_provider_module = os.environ.get('AMBARI_PROVIDER_MODULE')
# Non-root user setup commands
NR_USER_PROPERTY = "ambari-server.user"
-NR_USER_COMMENT = "Ambari user"
-NR_GET_OWNER_CMD = 'stat -c "%U" {0}'
-NR_USERADD_CMD = 'useradd -M --comment "{1}" ' \
- '--shell %s -d /var/lib/ambari-server/keys/ {0}' % utils.locate_file('nologin', '/sbin')
-NR_SET_USER_COMMENT_CMD = 'usermod -c "{0}" {1}'
NR_CHMOD_CMD = 'chmod {0} {1} {2}'
NR_CHOWN_CMD = 'chown {0} {1} {2}'
@@ -203,11 +202,6 @@ STACK_UPGRADE_HELPER_CMD = "{0}" + os.sep + "bin" + os.sep + "java -cp {1}" +\
" {3} {4} > " + SERVER_OUT_FILE + " 2>&1"
-VIEW_EXTRACT_CMD = "{0}" + os.sep + "bin" + os.sep + "java -cp {1}" +\
- os.pathsep + "{2} " +\
- "org.apache.ambari.server.view.ViewRegistry extract {3} " +\
- "> " + SERVER_OUT_FILE + " 2>&1"
-
ULIMIT_CMD = "ulimit -n"
SERVER_INIT_TIMEOUT = 5
@@ -265,8 +259,6 @@ SETUP_DB_CONNECT_TIMEOUT = 5
SETUP_DB_CONNECT_ATTEMPTS = 3
SETUP_DB_CMD = ['su', '-', 'postgres',
'--command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'" -v dbname="{3}"']
-UPGRADE_STACK_CMD = ['su', 'postgres',
- '--command=psql -f {0} -v stack_name="\'{1}\'" -v stack_version="\'{2}\'" -v dbname="{3}"']
CHANGE_OWNER_COMMAND = ['su', '-', 'postgres',
'--command=/var/lib/ambari-server/resources/scripts/change_owner.sh -d {0} -s {1} -o {2}']
@@ -276,7 +268,6 @@ PG_STATUS_RUNNING = utils.get_postgre_running_status(OS_TYPE)
PG_DEFAULT_PASSWORD = "bigdata"
SERVICE_CMD = "/usr/bin/env service"
PG_SERVICE_NAME = "postgresql"
-PG_HBA_DIR = utils.get_postgre_hba_dir(OS_FAMILY)
PG_ST_CMD = "%s %s status" % (SERVICE_CMD, PG_SERVICE_NAME)
if os.path.isfile("/usr/bin/postgresql-setup"):
@@ -285,12 +276,6 @@ else:
PG_INITDB_CMD = "%s %s initdb" % (SERVICE_CMD, PG_SERVICE_NAME)
PG_START_CMD = "%s %s start" % (SERVICE_CMD, PG_SERVICE_NAME)
-PG_RESTART_CMD = "%s %s restart" % (SERVICE_CMD, PG_SERVICE_NAME)
-PG_HBA_RELOAD_CMD = "%s %s reload" % (SERVICE_CMD, PG_SERVICE_NAME)
-
-PG_HBA_CONF_FILE = os.path.join(PG_HBA_DIR, "pg_hba.conf")
-PG_HBA_CONF_FILE_BACKUP = os.path.join(PG_HBA_DIR, "pg_hba_bak.conf.old")
-POSTGRESQL_CONF_FILE = os.path.join(PG_HBA_DIR, "postgresql.conf")
SERVER_VERSION_FILE_PATH = "server.version.file"
@@ -300,9 +285,6 @@ JDBC_HOSTNAME_PROPERTY = "server.jdbc.hostname"
JDBC_PORT_PROPERTY = "server.jdbc.port"
JDBC_POSTGRES_SCHEMA_PROPERTY = "server.jdbc.postgres.schema" # Only for postgres, defaults to same value as DB name
-VIEWS_DIR_PROPERTY = "views.dir"
-DEFAULT_VIEWS_DIR = "/var/lib/ambari-server/resources/views"
-
JDBC_USER_NAME_PROPERTY = "server.jdbc.user.name"
JDBC_PASSWORD_PROPERTY = "server.jdbc.user.passwd"
JDBC_PASSWORD_FILENAME = "password.dat"
@@ -333,9 +315,6 @@ PROMPT_DATABASE_OPTIONS = False
USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
PASSWORD_PATTERN = "^[a-zA-Z0-9_-]*$"
DATABASE_TYPES = ["postgres", "oracle", "mysql"]
-DATABASE_STORAGE_NAMES = ["Database", "Service", "Database"]
-DATABASE_PORTS = ["5432", "1521", "3306"]
-DATABASE_DRIVER_NAMES = ["org.postgresql.Driver", "oracle.jdbc.driver.OracleDriver", "com.mysql.jdbc.Driver"]
DATABASE_CONNECTION_STRINGS = [
"jdbc:postgresql://{0}:{1}/{2}",
"jdbc:oracle:thin:@{0}:{1}/{2}",
@@ -361,16 +340,6 @@ DATABASE_DROP_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-D
'/var/lib/ambari-server/resources/Ambari-DDL-Oracle-DROP.sql',
'/var/lib/ambari-server/resources/Ambari-DDL-MySQL-DROP.sql']
-JDBC_PROPERTIES_PREFIX = "server.jdbc.properties."
-DATABASE_JDBC_PROPERTIES = [
- [ ],
- [
- ["oracle.net.CONNECT_TIMEOUT", "2000"], # socket level timeout
- ["oracle.net.READ_TIMEOUT", "2000"], # socket level timeout
- ["oracle.jdbc.ReadTimeout", "8000"] # query fetch timeout
- ],
- [ ]
- ]
REGEX_IP_ADDRESS = "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
REGEX_HOSTNAME = "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$"
@@ -402,7 +371,6 @@ JDK_URL_PROPERTIES = ["jdk1.7.url", "jdk1.6.url"]
JCE_URL_PROPERTIES = ["jce_policy1.7.url", "jce_policy1.6.url"]
DEFAULT_JDK16_LOCATION = "/usr/jdk64/jdk1.6.0_31"
JDK_INDEX = 0
-JDK_VERSION_REs = ["(jdk.*)/jre", "Creating (jdk.*)/jre"]
CUSTOM_JDK_NUMBER = "3"
JDK_MIN_FILESIZE = 5000
JDK_INSTALL_DIR = "/usr/jdk64"
@@ -415,16 +383,8 @@ OS_TYPE_PROPERTY = "server.os_type"
OS_FAMILY_PROPERTY = "server.os_family"
GET_FQDN_SERVICE_URL = "server.fqdn.service.url"
-JDK_DOWNLOAD_CMD = "curl --create-dirs -o {0} {1}"
-JDK_DOWNLOAD_SIZE_CMD = "curl -I {0}"
-
-# use --no-same-owner when running as root to prevent uucp as the user (AMBARI-6478)
-UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}"
-
#JCE Policy files
JCE_POLICY_FILENAMES = ["UnlimitedJCEPolicyJDK7.zip", "jce_policy-6.zip"]
-JCE_DOWNLOAD_CMD = "curl -o {0} {1}"
-JCE_MIN_FILESIZE = 5000
DEFAULT_DB_NAME = "ambari"
@@ -575,30 +535,6 @@ NR_ADJUST_OWNERSHIP_LIST = [
### System interaction ###
-class FatalException(Exception):
- def __init__(self, code, reason):
- self.code = code
- self.reason = reason
-
- def __str__(self):
- return repr("Fatal exception: %s, exit code %s" % (self.reason, self.code))
-
-class NonFatalException(Exception):
- def __init__(self, reason):
- self.reason = reason
-
- def __str__(self):
- return repr("NonFatal exception: %s" % self.reason)
-
-
-def is_root():
- '''
- Checks effective UUID
- Returns True if a program is running under root-level privileges.
- '''
- return os.geteuid() == 0
-
-
def get_exec_path(cmd):
cmd = 'which {0}'.format(cmd)
ret, out, err = run_in_shell(cmd)
@@ -633,37 +569,6 @@ def run_os_command(cmd):
return process.returncode, stdoutdata, stderrdata
-#
-# Checks SELinux
-#
-def check_selinux():
- try:
- retcode, out, err = run_os_command(GET_SE_LINUX_ST_CMD)
- se_status = re.search('(disabled|enabled)', out).group(0)
- print "SELinux status is '" + se_status + "'"
- if se_status == SE_STATUS_DISABLED:
- return 0
- else:
- try:
- se_mode = re.search('(enforcing|permissive)', out).group(0)
- except AttributeError:
- err = "Error determining SELinux mode. Exiting."
- raise FatalException(1, err)
- print "SELinux mode is '" + se_mode + "'"
- if se_mode == SE_MODE_ENFORCING:
- print "Temporarily disabling SELinux"
- run_os_command(SE_SETENFORCE_CMD)
- print_warning_msg(
- "SELinux is set to 'permissive' mode and temporarily disabled.")
- ok = get_YN_input("OK to continue [y/n] (y)? ", True)
- if not ok:
- raise FatalException(1, None)
- return 0
- except OSError:
- print_warning_msg("Could not run {0}: OK".format(GET_SE_LINUX_ST_CMD))
- return 0
-
-
def read_ambari_user():
'''
Reads ambari user from properties file
@@ -729,30 +634,6 @@ def set_file_permissions(file, mod, user, recursive):
print_info_msg("File %s does not exist" % file)
-def create_custom_user():
- user = get_validated_string_input(
- "Enter user account for ambari-server daemon (root):",
- "root",
- "^[a-z_][a-z0-9_-]{1,31}$",
- "Invalid username.",
- False
- )
-
- print_info_msg("Trying to create user {0}".format(user))
- command = NR_USERADD_CMD.format(user, NR_USER_COMMENT)
- retcode, out, err = run_os_command(command)
- if retcode == 9: # 9 = username already in use
- print_info_msg("User {0} already exists, "
- "skipping user creation".format(user))
-
- elif retcode != 0: # fail
- print_warning_msg("Can't create user {0}. Command {1} "
- "finished with {2}: \n{3}".format(user, command, retcode, err))
- return retcode, None
-
- print_info_msg("User configuration is done.")
- return 0, user
-
def check_reverse_lookup():
"""
Check if host fqdn resolves to current host ip
@@ -767,115 +648,6 @@ def check_reverse_lookup():
pass
return False
-def check_ambari_user():
- try:
- user = read_ambari_user()
- create_user = False
- update_user_setting = False
- if user is not None:
- create_user = get_YN_input("Ambari-server daemon is configured to run under user '{0}'."
- " Change this setting [y/n] (n)? ".format(user), False)
- update_user_setting = create_user # Only if we will create another user
- else: # user is not configured yet
- update_user_setting = True # Write configuration anyway
- create_user = get_YN_input("Customize user account for ambari-server "
- "daemon [y/n] (n)? ", False)
- if not create_user:
- user = "root"
-
- if create_user:
- (retcode, user) = create_custom_user()
- if retcode != 0:
- return retcode
-
- if update_user_setting:
- write_property(NR_USER_PROPERTY, user)
-
- adjust_directory_permissions(user)
- except OSError as e:
- print_error_msg("Failed: %s" % e.strerror)
- return 4
- except Exception as e:
- print_error_msg("Unexpected error %s" % e)
- return 1
- return 0
-
-### Postgres ###
-
-
-def configure_pg_hba_ambaridb_users():
- args = optparse.Values()
- configure_database_username_password(args)
-
- with open(PG_HBA_CONF_FILE, "a") as pgHbaConf:
- pgHbaConf.write("\n")
- pgHbaConf.write("local all " + args.database_username +
- ",mapred md5")
- pgHbaConf.write("\n")
- pgHbaConf.write("host all " + args.database_username +
- ",mapred 0.0.0.0/0 md5")
- pgHbaConf.write("\n")
- pgHbaConf.write("host all " + args.database_username +
- ",mapred ::/0 md5")
- pgHbaConf.write("\n")
- retcode, out, err = run_os_command(PG_HBA_RELOAD_CMD)
- if not retcode == 0:
- raise FatalException(retcode, err)
-
-
-def configure_pg_hba_postgres_user():
- postgresString = "all postgres"
- for line in fileinput.input(PG_HBA_CONF_FILE, inplace=1):
- print re.sub('all\s*all', postgresString, line),
- os.chmod(PG_HBA_CONF_FILE, 0644)
-
-
-def configure_postgresql_conf():
- listenAddress = "listen_addresses = '*' #"
- for line in fileinput.input(POSTGRESQL_CONF_FILE, inplace=1):
- print re.sub('#+listen_addresses.*?(#|$)', listenAddress, line),
- os.chmod(POSTGRESQL_CONF_FILE, 0644)
-
-
-def configure_postgres():
- if os.path.isfile(PG_HBA_CONF_FILE):
- if not os.path.isfile(PG_HBA_CONF_FILE_BACKUP):
- shutil.copyfile(PG_HBA_CONF_FILE, PG_HBA_CONF_FILE_BACKUP)
- else:
- #Postgres has been configured before, must not override backup
- print "Backup for pg_hba found, reconfiguration not required"
- return 0, "", ""
- configure_pg_hba_postgres_user()
- configure_pg_hba_ambaridb_users()
- os.chmod(PG_HBA_CONF_FILE, 0644)
- configure_postgresql_conf()
- #restart postgresql if already running
- pg_status, retcode, out, err = get_postgre_status()
- if pg_status == PG_STATUS_RUNNING:
- retcode, out, err = restart_postgres()
- return retcode, out, err
- return 0, "", ""
-
-
-def restart_postgres():
- print "Restarting PostgreSQL"
- process = subprocess.Popen(PG_RESTART_CMD.split(' '),
- stdout=subprocess.PIPE,
- stdin=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
- time.sleep(5)
- result = process.poll()
- if result is None:
- print_info_msg("Killing restart PostgresSQL process")
- process.kill()
- pg_status, retcode, out, err = get_postgre_status()
- # SUSE linux set status of stopped postgresql proc to unused
- if pg_status == "unused" or pg_status == "stopped":
- print_info_msg("PostgreSQL is stopped. Restarting ...")
- retcode, out, err = run_os_command(PG_START_CMD)
- return retcode, out, err
- return 0, "", ""
def write_property(key, value):
@@ -1057,35 +829,6 @@ def check_postgre_up():
return pg_status, retcode, out, err
-def get_validated_db_name(database_name):
- return get_validated_string_input(
- DATABASE_STORAGE_NAMES[DATABASE_INDEX] + " Name ("
- + database_name + "): ",
- database_name,
- ".*",
- "Invalid " + DATABASE_STORAGE_NAMES[DATABASE_INDEX] + " name.",
- False
- )
-
-def get_validated_db_schema(postgres_schema):
- return get_validated_string_input(
- "Postgres schema (" + postgres_schema + "): ",
- postgres_schema,
- "^[a-zA-Z0-9_\-]*$",
- "Invalid schema name.",
- False, allowEmpty=True
- )
-
-def get_validated_service_name(service_name, index):
- return get_validated_string_input(
- ORACLE_DB_ID_TYPES[index] + " (" + service_name + "): ",
- service_name,
- ".*",
- "Invalid " + ORACLE_DB_ID_TYPES[index] + ".",
- False
- )
-
-
def read_password(passwordDefault=PG_DEFAULT_PASSWORD,
passwordPattern=PASSWORD_PATTERN,
passwordPrompt=None,
@@ -1122,244 +865,6 @@ def get_pass_file_path(conf_file):
JDBC_PASSWORD_FILENAME)
-# Set database properties to default values
-def load_default_db_properties(args):
- args.persistence_type = 'local'
- args.dbms = DATABASE_TYPES[DATABASE_INDEX]
- args.database_host = "localhost"
- args.database_port = DATABASE_PORTS[DATABASE_INDEX]
- args.database_name = DEFAULT_DB_NAME
- args.postgres_schema = DEFAULT_DB_NAME
- args.database_username = "ambari"
- args.database_password = "bigdata"
- args.sid_or_sname = "sname"
- pass
-
-
-# Ask user for database conenction properties
-def prompt_db_properties(args):
- global DATABASE_INDEX
-
- if PROMPT_DATABASE_OPTIONS:
- load_default_db_properties(args)
- ok = get_YN_input("Enter advanced database configuration [y/n] (n)? ", False)
- if ok:
-
- print "=============================================================================="
- print "Choose one of the following options:"
-
- database_num = str(DATABASE_INDEX + 1)
- database_num = get_validated_string_input(
- "[1] - PostgreSQL (Embedded)\n[2] - Oracle\n[3] - MySQL\n[4] - PostgreSQL\n"
- "==============================================================================\n"
- "Enter choice (" + database_num + "): ",
- database_num,
- "^[1234]$",
- "Invalid number.",
- False
- )
-
- if int(database_num) == 1:
- args.persistence_type = 'local'
- args.database_index = 0
- else:
- args.persistence_type = 'remote'
- selected_db_option = int(database_num)
-
- if selected_db_option == 2:
- args.database_index = 1
- elif selected_db_option == 3:
- args.database_index = 2
- elif selected_db_option == 4:
- args.database_index = 0
- else:
- print_info_msg('Unknown db option, default to embbeded postgres.')
- args.database_index = 0
- pass
- pass
-
- DATABASE_INDEX = args.database_index
- args.dbms = DATABASE_TYPES[args.database_index]
-
- if args.persistence_type != 'local':
- args.database_host = get_validated_string_input(
- "Hostname (" + args.database_host + "): ",
- args.database_host,
- "^[a-zA-Z0-9.\-]*$",
- "Invalid hostname.",
- False
- )
-
- args.database_port = DATABASE_PORTS[DATABASE_INDEX]
- args.database_port = get_validated_string_input(
- "Port (" + args.database_port + "): ",
- args.database_port,
- "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$",
- "Invalid port.",
- False
- )
-
- if args.dbms == "oracle":
- # Oracle uses service name or service id
- idType = "1"
- idType = get_validated_string_input(
- "Select Oracle identifier type:\n1 - " + ORACLE_DB_ID_TYPES[0] +
- "\n2 - " + ORACLE_DB_ID_TYPES[1] + "\n(" + idType + "): ",
- idType,
- "^[12]$",
- "Invalid number.",
- False
- )
-
- if idType == "2":
- args.sid_or_sname = "sid"
-
- IDTYPE_INDEX = int(idType) - 1
- args.database_name = get_validated_service_name(args.database_name,
- IDTYPE_INDEX)
- elif args.dbms in ["mysql", "postgres"]:
- args.database_name = get_validated_db_name(args.database_name)
-
- if args.dbms in ["postgres", ]:
- args.postgres_schema = get_validated_db_schema(args.postgres_schema)
- else:
- # other DB types
- pass
- pass
- else:
- args.database_host = "localhost"
- args.database_port = DATABASE_PORTS[DATABASE_INDEX]
-
- args.database_name = get_validated_db_name(args.database_name)
- if args.dbms in ["postgres", ]:
- args.postgres_schema = get_validated_db_schema(args.postgres_schema)
-
- # Username is common for Oracle/MySQL/Postgres
- args.database_username = get_validated_string_input(
- 'Username (' + args.database_username + '): ',
- args.database_username,
- USERNAME_PATTERN,
- "Invalid characters in username. Start with _ or alpha "
- "followed by alphanumeric or _ or - characters",
- False
- )
- args.database_password = configure_database_password(True)
-
- print_info_msg('Using database options: {database},{host},{port},{name},{schema},{user},{password}'.format(
- database=args.dbms,
- host=args.database_host,
- port=args.database_port,
- name=args.database_name,
- schema=args.postgres_schema,
- user=args.database_username,
- password=args.database_password
- ))
-
-# extract the system views
-def extract_views():
- jdk_path = find_jdk()
- if jdk_path is None:
- print_error_msg("No JDK found, please run the \"setup\" "
- "command to install a JDK automatically or install any "
- "JDK manually to " + JDK_INSTALL_DIR)
- return 1
-
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return -1
-
- if not VIEWS_DIR_PROPERTY in properties.keys():
- vdir = DEFAULT_VIEWS_DIR
- else:
- vdir = properties.get_property(VIEWS_DIR_PROPERTY)
-
- files = [f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir,f))]
- for f in files:
-
- command = VIEW_EXTRACT_CMD.format(jdk_path, get_conf_dir(),
- get_ambari_classpath(), os.path.join(vdir,f))
-
- retcode, stdout, stderr = run_os_command(command)
- if retcode == 0:
- sys.stdout.write(f + "\n")
- elif retcode == 2:
- sys.stdout.write("Error extracting " + f + "\n")
- else:
- sys.stdout.write(".")
- sys.stdout.flush()
-
- print_info_msg("Return code from extraction of view archive " + f + ": " +
- str(retcode))
-
- sys.stdout.write("\n")
- return 0
-
-# Store set of properties for remote database connection
-def store_remote_properties(args):
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return -1
-
- isSecure = get_is_secure(properties)
-
- properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "remote")
-
- properties.process_pair(JDBC_DATABASE_PROPERTY, args.dbms)
- properties.process_pair(JDBC_HOSTNAME_PROPERTY, args.database_host)
- properties.process_pair(JDBC_PORT_PROPERTY, args.database_port)
- properties.process_pair(JDBC_DATABASE_NAME_PROPERTY, args.database_name)
- if args.dbms == "postgres":
- properties.process_pair(JDBC_POSTGRES_SCHEMA_PROPERTY, args.postgres_schema)
- properties.process_pair(JDBC_DRIVER_PROPERTY, DATABASE_DRIVER_NAMES[DATABASE_INDEX])
- # fully qualify the hostname to make sure all the other hosts can connect
- # to the jdbc hostname since its passed onto the agents for RCA
- jdbc_hostname = args.database_host
- if (args.database_host == "localhost"):
- jdbc_hostname = socket.getfqdn().lower()
-
- connectionStringFormat = DATABASE_CONNECTION_STRINGS
- if args.sid_or_sname == "sid":
- connectionStringFormat = DATABASE_CONNECTION_STRINGS_ALT
- properties.process_pair(JDBC_URL_PROPERTY, connectionStringFormat[DATABASE_INDEX].format(jdbc_hostname, args.database_port, args.database_name))
- properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username)
- properties.process_pair(JDBC_PASSWORD_PROPERTY,
- store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
-
- # save any other defined properties to pass to JDBC
- if DATABASE_INDEX < len(DATABASE_JDBC_PROPERTIES):
- for pair in DATABASE_JDBC_PROPERTIES[DATABASE_INDEX]:
- properties.process_pair(JDBC_PROPERTIES_PREFIX + pair[0], pair[1])
-
- if isSecure:
- encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password)
- if encrypted_password != args.database_password:
- properties.process_pair(JDBC_PASSWORD_PROPERTY, encrypted_password)
- pass
-
- properties.process_pair(JDBC_RCA_DRIVER_PROPERTY, DATABASE_DRIVER_NAMES[DATABASE_INDEX])
- properties.process_pair(JDBC_RCA_URL_PROPERTY, connectionStringFormat[DATABASE_INDEX].format(jdbc_hostname, args.database_port, args.database_name))
- properties.process_pair(JDBC_RCA_USER_NAME_PROPERTY, args.database_username)
- properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY,
- store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
- if isSecure:
- encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password)
- if encrypted_password != args.database_password:
- properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY, encrypted_password)
- pass
-
- conf_file = properties.fileName
-
- try:
- properties.store(open(conf_file, "w"))
- except Exception, e:
- print 'Could not write ambari config file "%s": %s' % (conf_file, e)
- return -1
-
- return 0
-
-
# Initialize remote database schema
def setup_remote_db(args):
@@ -1447,22 +952,6 @@ def get_remote_script_line(args, scriptPath, forPrint=True):
return None
-def configure_database_password(showDefault=True):
- passwordDefault = PG_DEFAULT_PASSWORD
- if showDefault:
- passwordPrompt = 'Enter Database Password (' + passwordDefault + '): '
- else:
- passwordPrompt = 'Enter Database Password: '
- passwordPattern = "^[a-zA-Z0-9_-]*$"
- passwordDescr = "Invalid characters in password. Use only alphanumeric or "\
- "_ or - characters"
-
- password = read_password(passwordDefault, passwordPattern, passwordPrompt,
- passwordDescr)
-
- return password
-
-
def get_ambari_version(properties):
"""
:param properties: Ambari properties
@@ -1564,71 +1053,6 @@ def configure_database_username_password(args):
print_error_msg("Connection properties not set in config file.")
-# Check if jdbc user is changed
-def is_jdbc_user_changed(args):
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return None
-
- previos_user = properties[JDBC_USER_NAME_PROPERTY]
- new_user = args.database_username
-
- if previos_user and new_user:
- if previos_user != new_user:
- return True
- else:
- return False
-
- return None
-
-
-# Store local database connection properties
-def store_local_properties(args):
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return -1
-
- isSecure = get_is_secure(properties)
-
- properties.removeOldProp(JDBC_DATABASE_PROPERTY)
- properties.removeOldProp(JDBC_DATABASE_NAME_PROPERTY)
- properties.removeOldProp(JDBC_POSTGRES_SCHEMA_PROPERTY)
- properties.removeOldProp(JDBC_HOSTNAME_PROPERTY)
- properties.removeOldProp(JDBC_RCA_DRIVER_PROPERTY)
- properties.removeOldProp(JDBC_RCA_URL_PROPERTY)
- properties.removeOldProp(JDBC_PORT_PROPERTY)
- properties.removeOldProp(JDBC_DRIVER_PROPERTY)
- properties.removeOldProp(JDBC_URL_PROPERTY)
-
- # Store the properties
- properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "local")
- properties.process_pair(JDBC_DATABASE_PROPERTY, args.dbms)
- properties.process_pair(JDBC_DATABASE_NAME_PROPERTY, args.database_name)
- if args.dbms == "postgres":
- properties.process_pair(JDBC_POSTGRES_SCHEMA_PROPERTY, args.postgres_schema)
- properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username)
- properties.process_pair(JDBC_PASSWORD_PROPERTY,
- store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
-
- if isSecure:
- encrypted_password = encrypt_password(JDBC_RCA_PASSWORD_ALIAS, args.database_password)
- if args.database_password != encrypted_password:
- properties.process_pair(JDBC_PASSWORD_PROPERTY, encrypted_password)
- pass
- pass
-
- conf_file = properties.fileName
-
- try:
- properties.store(open(conf_file, "w"))
- except Exception, e:
- print 'Unable to write ambari.properties configuration file "%s": %s' % (conf_file, e)
- return -1
- return 0
-
-
# Load ambari properties and return dict with values
def get_ambari_properties():
conf_file = find_properties_file()
@@ -1722,40 +1146,6 @@ def search_file(filename, search_path, pathsep=os.pathsep):
return None
-def dlprogress(base_name, count, blockSize, totalSize):
- percent = int(count * blockSize * 100 / totalSize)
-
- if (totalSize < blockSize):
- sys.stdout.write("\r" + base_name + "... %d%%" % (100))
- else:
- sys.stdout.write("\r" + base_name + "... %d%% (%.1f MB of %.1f MB)" % (
- percent, count * blockSize / 1024 / 1024.0, totalSize / 1024 / 1024.0))
-
- if (percent == 100 or totalSize < blockSize):
- sys.stdout.write("\n")
- sys.stdout.flush()
-
-
-def track_jdk(base_name, url, local_name):
- u = urllib2.urlopen(url)
- h = u.info()
- totalSize = int(h["Content-Length"])
- fp = open(local_name, "wb")
- blockSize = 8192
- count = 0
- while True:
- chunk = u.read(blockSize)
- if not chunk:
- break
- fp.write(chunk)
- count += 1
-
- dlprogress(base_name, count, blockSize, totalSize)
-
- fp.flush()
- fp.close()
-
-
def install_jce_manualy(args):
properties = get_ambari_properties()
if properties == -1:
@@ -1782,279 +1172,10 @@ def install_jce_manualy(args):
return 1
-#
-# Downloads the JDK
-#
-def download_jdk(args):
- global JDK_INDEX
- properties = get_ambari_properties()
- if properties == -1:
- err = "Error getting ambari properties"
- raise FatalException(-1, err)
- conf_file = properties.fileName
- ok = False
- jcePolicyWarn = "JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos," \
- "please make sure JCE Unlimited Strength Jurisdiction Policy Files are valid on all hosts."
- if args.java_home:
- if not os.path.exists(args.java_home) or not os.path.isfile(os.path.join(args.java_home, "bin", "java")):
- err = "Path to java home " + args.java_home + " or java binary file does not exists"
- raise FatalException(1, err)
-
- print_warning_msg("JAVA_HOME " + args.java_home + " must be valid on ALL hosts")
- print_warning_msg(jcePolicyWarn)
- write_property(JAVA_HOME_PROPERTY, args.java_home)
- remove_property(JDK_NAME_PROPERTY)
- remove_property(JCE_NAME_PROPERTY)
- return 0
- else:
- if get_JAVA_HOME():
- change_jdk = get_YN_input("Do you want to change Oracle JDK [y/n] (n)? ", False)
- if not change_jdk:
- return 0
-
- try:
- resources_dir = properties[RESOURCES_DIR_PROPERTY]
- except (KeyError), e:
- err = 'Property ' + str(e) + ' is not defined at ' + conf_file
- raise FatalException(1, err)
- jdk_num = str(JDK_INDEX + 1)
- jdk_num = get_validated_string_input(
-"""[1] - Oracle JDK 1.7 + Java Cryptography Extension (JCE) Policy Files 7
-[2] - Oracle JDK 1.6 + Java Cryptography Extension (JCE) Policy Files 6
-[3] - Custom JDK
-==============================================================================
-Enter choice (""" + jdk_num + "):",
- jdk_num,
- "^[123]$",
- "Invalid number.",
- False
- )
-
- if jdk_num == CUSTOM_JDK_NUMBER:
- print_warning_msg("JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts.")
- print_warning_msg(jcePolicyWarn)
- args.java_home = get_validated_string_input("Path to JAVA_HOME: ", None, None, None, False, False)
- if not os.path.exists(args.java_home) or not os.path.isfile(os.path.join(args.java_home, "bin", "java")):
- err = "Java home path or java binary file is unavailable. Please put correct path to java home."
- raise FatalException(1, err)
- print "Validating JDK on Ambari Server...done."
- write_property(JAVA_HOME_PROPERTY, args.java_home)
- remove_property(JDK_NAME_PROPERTY)
- remove_property(JCE_NAME_PROPERTY)
- return 0
-
- JDK_INDEX = int(jdk_num) - 1
- JDK_FILENAME = JDK_NAMES[JDK_INDEX]
- JDK_URL_PROPERTY = JDK_URL_PROPERTIES[JDK_INDEX]
-
- try:
- jdk_url = properties[JDK_URL_PROPERTY]
- except (KeyError), e:
- err = 'Property ' + str(e) + ' is not defined at ' + conf_file
- raise FatalException(1, err)
- dest_file = resources_dir + os.sep + JDK_FILENAME
- if os.path.exists(dest_file):
- print "JDK already exists, using " + dest_file
- else:
- ok = get_YN_input("To download the Oracle JDK and the Java Cryptography Extension (JCE) "
- "Policy Files you must accept the "
- "license terms found at "
- "http://www.oracle.com/technetwork/java/javase/"
- "terms/license/index.html and not accepting will "
- "cancel the Ambari Server setup and you must install the JDK and JCE "
- "files manually.\nDo you accept the "
- "Oracle Binary Code License Agreement [y/n] (y)? ", True)
- if not ok:
- print 'Exiting...'
- sys.exit(1)
-
- print 'Downloading JDK from ' + jdk_url + ' to ' + dest_file
- jdk_download_fail_msg = " Failed to download JDK: {0}. Please check that Oracle " \
- "JDK is available at {1}. Also you may specify JDK file " \
- "location in local filesystem using --jdk-location command " \
- "line argument.".format("{0}", jdk_url)
- try:
- size_command = JDK_DOWNLOAD_SIZE_CMD.format(jdk_url)
- #Get Header from url,to get file size then
- retcode, out, err = run_os_command(size_command)
- if out.find("Content-Length") == -1:
- err = jdk_download_fail_msg.format("Request header doesn't contain Content-Length")
- raise FatalException(1, err)
- start_with = int(out.find("Content-Length") + len("Content-Length") + 2)
- end_with = out.find("\r\n", start_with)
- src_size = int(out[start_with:end_with])
- print 'JDK distribution size is ' + str(src_size) + ' bytes'
- file_exists = os.path.isfile(dest_file)
- file_size = -1
- if file_exists:
- file_size = os.stat(dest_file).st_size
- if file_exists and file_size == src_size:
- print_info_msg("File already exists")
- else:
- track_jdk(JDK_FILENAME, jdk_url, dest_file)
- print 'Successfully downloaded JDK distribution to ' + dest_file
- except FatalException:
- raise
- except Exception, e:
- err = jdk_download_fail_msg.format(str(e))
- raise FatalException(1, err)
- downloaded_size = os.stat(dest_file).st_size
- if downloaded_size != src_size or downloaded_size < JDK_MIN_FILESIZE:
- err = 'Size of downloaded JDK distribution file is ' \
- + str(downloaded_size) + ' bytes, it is probably \
- damaged or incomplete'
- raise FatalException(1, err)
-
- try:
- out = install_jdk(dest_file)
- jdk_version = re.search(JDK_VERSION_REs[JDK_INDEX], out).group(1)
- write_property(JDK_NAME_PROPERTY, JDK_FILENAME)
- except Exception, e:
- print "Installation of JDK has failed: %s\n" % e
- file_exists = os.path.isfile(dest_file)
- if file_exists:
- ok = get_YN_input("JDK found at "+dest_file+". "
- "Would you like to re-download the JDK [y/n] (y)? ", True)
- if not ok:
- err = "Unable to install JDK. Please remove JDK file found at " + \
- dest_file + " and re-run Ambari Server setup"
- raise FatalException(1, err)
- else:
- track_jdk(JDK_FILENAME, jdk_url, dest_file)
- print 'Successfully re-downloaded JDK distribution to ' + dest_file
- try:
- out = install_jdk(dest_file)
- jdk_version = re.search(JDK_VERSION_REs[JDK_INDEX], out).group(1)
- write_property(JDK_NAME_PROPERTY, JDK_FILENAME)
- except Exception, e:
- print "Installation of JDK was failed: %s\n" % e
- err = "Unable to install JDK. Please remove JDK, file found at " + \
- dest_file + " and re-run Ambari Server setup"
- raise FatalException(1, err)
-
- else:
- err = "Unable to install JDK. File " + dest_file + " does not exist, " \
- "please re-run Ambari Server setup"
- raise FatalException(1, err)
-
- print "Successfully installed JDK to {0}/{1}".\
- format(JDK_INSTALL_DIR, jdk_version)
- write_property(JAVA_HOME_PROPERTY, "{0}/{1}".
- format(JDK_INSTALL_DIR, jdk_version))
-
- try:
- download_jce_policy(properties, ok)
- except FatalException as e:
- print "JCE Policy files are required for secure HDP setup. Please ensure " \
- " all hosts have the JCE unlimited strength policy 6, files."
- print_error_msg("Failed to download JCE policy files:")
- if e.reason is not None:
- print_error_msg("\nREASON: {0}".format(e.reason))
- # TODO: We don't fail installation if download_jce_policy fails. Is it OK?
- return 0
-
-
-def download_jce_policy(properties, accpeted_bcl):
- JCE_URL_PROPERTY = JCE_URL_PROPERTIES[JDK_INDEX]
- JCE_POLICY_FILENAME = JCE_POLICY_FILENAMES[JDK_INDEX]
- try:
- jce_url = properties[JCE_URL_PROPERTY]
- resources_dir = properties[RESOURCES_DIR_PROPERTY]
- except KeyError, e:
- err = 'Property ' + str(e) + ' is not defined in properties file'
- raise FatalException(1, err)
- dest_file = resources_dir + os.sep + JCE_POLICY_FILENAME
- if not os.path.exists(dest_file):
- print 'Downloading JCE Policy archive from ' + jce_url + ' to ' + dest_file
- jce_download_fail_msg = " Failed to download JCE Policy archive : {0}. " \
- "Please check that JCE Policy archive is available at {1} . "
- try:
- size_command = JDK_DOWNLOAD_SIZE_CMD.format(jce_url)
- #Get Header from url,to get file size then
- retcode, out, err = run_os_command(size_command)
- if out.find("Content-Length") == -1:
- err = jce_download_fail_msg.format(
- "Request header doesn't contain Content-Length")
- raise FatalException(1, err)
- start_with = int(out.find("Content-Length") + len("Content-Length") + 2)
- end_with = out.find("\r\n", start_with)
- src_size = int(out[start_with:end_with])
- print_info_msg('JCE zip distribution size is ' + str(src_size) + ' bytes')
- file_exists = os.path.isfile(dest_file)
- file_size = -1
- if file_exists:
- file_size = os.stat(dest_file).st_size
- if file_exists and file_size == src_size:
- print_info_msg("File already exists")
- else:
- #BCL license before download
- jce_download_cmd = JCE_DOWNLOAD_CMD.format(dest_file, jce_url)
- print_info_msg("JCE download cmd: " + jce_download_cmd)
- if accpeted_bcl:
- retcode, out, err = run_os_command(jce_download_cmd)
- if retcode == 0:
- write_property(JCE_NAME_PROPERTY, JCE_POLICY_FILENAME)
- print 'Successfully downloaded JCE Policy archive to ' + dest_file
- else:
- raise FatalException(1, err)
- else:
- ok = get_YN_input("To download the JCE Policy files you must "
- "accept the license terms found at "
- "http://www.oracle.com/technetwork/java/javase"
- "/terms/license/index.html"
- "Not accepting will result in errors when "
- "configuring Kerberos security. \nDo you accept the "
- "Oracle Binary Code License Agreement [y/n] (y)? ", True)
- if ok:
- retcode, out, err = run_os_command(jce_download_cmd)
- if retcode == 0:
- write_property(JCE_NAME_PROPERTY, JCE_POLICY_FILENAME)
- print 'Successfully downloaded JCE Policy archive to ' + dest_file
- else:
- raise FatalException(1, None)
- except FatalException:
- raise
- except Exception, e:
- err = 'Failed to download JCE Policy archive: ' + str(e)
- raise FatalException(1, err)
- downloaded_size = os.stat(dest_file).st_size
- if downloaded_size != src_size or downloaded_size < JCE_MIN_FILESIZE:
- err = 'Size of downloaded JCE Policy archive is ' \
- + str(downloaded_size) + ' bytes, it is probably \
- damaged or incomplete'
- raise FatalException(1, err)
- else:
- write_property(JCE_NAME_PROPERTY, JCE_POLICY_FILENAME)
- print "JCE Policy archive already exists, using " + dest_file
-
-
class RetCodeException(Exception):
pass
-def install_jdk(dest_file):
- print "Installing JDK to {0}".format(JDK_INSTALL_DIR)
- retcode, out, err = run_os_command(CREATE_JDK_DIR_CMD)
- retcode, out, err = run_os_command(MAKE_FILE_EXECUTABLE_CMD.format(JDK_INSTALL_DIR))
- savedPath = os.getcwd()
- os.chdir(JDK_INSTALL_DIR)
-
- if dest_file.endswith(".bin"):
- retcode, out, err = run_os_command(MAKE_FILE_EXECUTABLE_CMD.format(dest_file))
- retcode, out, err = run_os_command(dest_file + ' -noregister')
- elif dest_file.endswith(".gz"):
- retcode, out, err = run_os_command(UNTAR_JDK_ARCHIVE.format(dest_file))
- else:
- err = "JDK installation failed.Unknown file mask."
- raise FatalException(1, err)
-
- os.chdir(savedPath)
- if retcode != 0:
- err = "Installation of JDK returned exit code %s" % retcode
- raise FatalException(retcode, err)
- return out
-
-
#
# Configures the OS settings in ambari properties.
#
@@ -2191,241 +1312,6 @@ def wait_popen(popen, timeout=0):
time.sleep(1)
return popen.poll()
-def check_jdbc_drivers(args):
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return -1
-
- result = find_jdbc_driver(args)
-
- msg = 'Before starting Ambari Server, ' \
- 'you must copy the {0} JDBC driver JAR file to {1}.'.format(
- DATABASE_FULL_NAMES[args.dbms],
- JAVA_SHARE_PATH)
-
- if result == -1:
- if SILENT:
- print_error_msg(msg)
- raise FatalException(-1, msg)
- else:
- print_warning_msg(msg)
- raw_input(PRESS_ENTER_MSG)
- result = find_jdbc_driver(args)
- if result == -1:
- print_error_msg(msg)
- raise FatalException(-1, msg)
-
- # Check if selected RDBMS requires drivers to copy
- if type(result) is not int:
- print 'Copying JDBC drivers to server resources...'
- try:
- resources_dir = properties[RESOURCES_DIR_PROPERTY]
- except KeyError:
- print_error_msg("There is no value for " + RESOURCES_DIR_PROPERTY + "in " + AMBARI_PROPERTIES_FILE)
- return -1
-
- db_name = DATABASE_FULL_NAMES[args.dbms].lower()
- jdbc_symlink = os.path.join(resources_dir, db_name + "-jdbc-driver.jar")
- db_default_driver_path = os.path.join(JAVA_SHARE_PATH, JDBC_DB_DEFAULT_DRIVER[db_name])
-
- if os.path.lexists(jdbc_symlink):
- os.remove(jdbc_symlink)
-
- copy_status = copy_files(result, resources_dir)
-
- if not copy_status == 0:
- raise FatalException(-1, "Failed to copy JDBC drivers to server resources")
-
- if db_default_driver_path in result:
- os.symlink(os.path.join(resources_dir, JDBC_DB_DEFAULT_DRIVER[db_name]), jdbc_symlink)
-
- return 0
-
-
-def verify_setup_allowed():
- properties = get_ambari_properties()
- if properties == -1:
- print_error_msg("Error getting ambari properties")
- return -1
-
- isSecure = get_is_secure(properties)
- (isPersisted, masterKeyFile) = get_is_persisted(properties)
- if isSecure and not isPersisted and SILENT:
- print "ERROR: Cannot run silent 'setup' with password encryption enabled " \
- "and Master Key not persisted."
- print "Ambari Server 'setup' exiting."
- return 1
- return 0
-
-
-#
-# Setup the Ambari Server.
-#
-def setup(args):
- retcode = verify_setup_allowed()
- if not retcode == 0:
- raise FatalException(1, None)
-
- if not is_root():
- err = 'Ambari-server setup should be run with '\
- 'root-level privileges'
- raise FatalException(4, err)
-
- # proceed jdbc properties if they were set
- if args.jdbc_driver is not None and args.jdbc_db is not None:
- proceedJDBCProperties(args)
- return
-
- print 'Checking SELinux...'
- retcode = check_selinux()
- if not retcode == 0:
- err = 'Failed to disable SELinux. Exiting.'
- raise FatalException(retcode, err)
-
- # Create ambari user, if needed
- retcode = check_ambari_user()
-
- if not retcode == 0:
- err = 'Failed to create user. Exiting.'
- raise FatalException(retcode, err)
-
- print 'Checking firewall...'
- firewall_obj = Firewall().getFirewallObject()
- firewall_on = firewall_obj.check_iptables()
- if firewall_obj.stderrdata and len(firewall_obj.stderrdata) > 0:
- print firewall_obj.stderrdata
- if firewall_on:
- print_warning_msg("%s is running. Confirm the necessary Ambari ports are accessible. " %
- firewall_obj.FIREWALL_SERVICE_NAME +
- "Refer to the Ambari documentation for more details on ports.")
- ok = get_YN_input("OK to continue [y/n] (y)? ", True)
- if not ok:
- raise FatalException(1, None)
-
-
-
-
- # proceed jdbc properties if they were set
- if args.jdbc_driver is not None and args.jdbc_db is not None:
- proceedJDBCProperties(args)
-
- print 'Checking JDK...'
- try:
- download_jdk(args)
- except FatalException as e:
- err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
- raise FatalException(e.code, err)
-
- print 'Completing setup...'
- retcode = configure_os_settings()
- if not retcode == 0:
- err = 'Configure of OS settings in ambari.properties failed. Exiting.'
- raise FatalException(retcode, err)
-
- print 'Configuring database...'
- prompt_db_properties(args)
-
- #DB setup should be done last after doing any setup.
-
- if is_local_database(args):
- #check if jdbc user is changed
- is_user_changed = is_jdbc_user_changed(args)
-
- print 'Default properties detected. Using built-in database.'
- store_local_properties(args)
-
- print 'Checking PostgreSQL...'
- pg_status, retcode, out, err = check_postgre_up()
- if not retcode == 0:
- err = 'Unable to start PostgreSQL server. Status {0}. {1}.' \
- ' Exiting'.format(pg_status, err)
- raise FatalException(retcode, err)
-
- print 'Configuring local database...'
- retcode, outdata, errdata = setup_db(args)
- if not retcode == 0:
- err = 'Running database init script was failed. {0}. Exiting.'.format(errdata)
- raise FatalException(retcode, err)
-
- if is_user_changed:
- #remove backup for pg_hba in order to reconfigure postgres
- remove_file(PG_HBA_CONF_FILE_BACKUP)
-
- print 'Configuring PostgreSQL...'
- retcode, out, err = configure_postgres()
- if not retcode == 0:
- err = 'Unable to configure PostgreSQL server. {0} Exiting'.format(err)
- raise FatalException(retcode, err)
-
- else:
- retcode = store_remote_properties(args)
- if retcode != 0:
- err = 'Unable to save config file'
- raise FatalException(retcode, err)
-
- check_jdbc_drivers(args)
-
- print 'Configuring remote database connection properties...'
- retcode = setup_remote_db(args)
- if retcode == -1:
- err = "Remote database setup aborted."
- raise NonFatalException(err)
-
- if not retcode == 0:
- err = 'Error while configuring connection properties. Exiting'
- raise FatalException(retcode, err)
- check_jdbc_drivers(args)
-
- print 'Extracting system views...'
- retcode = extract_views()
- if not retcode == 0:
- err = 'Error while extracting system views. Exiting'
- raise FatalException(retcode, err)
-
- # we've already done this, but new files were created so run it one time.
- adjust_directory_permissions(read_ambari_user())
-
-
-def proceedJDBCProperties(args):
- if not os.path.isfile(args.jdbc_driver):
- err = "File {0} does not exist!".format(args.jdbc_driver)
- raise FatalException(1, err)
-
- if args.jdbc_db not in JDBC_DB_OPTION_VALUES:
- err = "Unsupported database name {0}. Please see help for more information.".format(args.jdbc_db)
- raise FatalException(1, err)
-
- properties = get_ambari_properties()
- if properties == -1:
- err = "Error getting ambari properties"
- raise FatalException(-1, err)
- conf_file = properties.fileName
-
- try:
- resources_dir = properties[RESOURCES_DIR_PROPERTY]
- except (KeyError), e:
- err = 'Property ' + str(e) + ' is not defined at ' + conf_file
- raise FatalException(1, err)
-
- symlink_name = args.jdbc_db + "-jdbc-driver.jar"
- jdbc_symlink = os.path.join(resources_dir, symlink_name)
- path, jdbc_name = os.path.split(args.jdbc_driver)
-
- if os.path.lexists(jdbc_symlink):
- os.remove(jdbc_symlink)
-
- if not os.path.isfile(os.path.join(resources_dir, jdbc_name)):
- try:
- shutil.copy(args.jdbc_driver, resources_dir)
- except Exception, e:
- err = "Can not copy file {0} to {1} due to: {2} . Please check file " \
- "permissions and free disk space.".format(args.jdbc_driver, resources_dir, e)
- raise FatalException(1, err)
-
- os.symlink(os.path.join(resources_dir,jdbc_name), jdbc_symlink)
- print "JDBC driver was successfully initialized ."
-
#
# Resets the Ambari Server.
@@ -4511,6 +3397,8 @@ def main():
and options.database_password is not None):
parser.error('All database options should be set. Please see help for the options.')
+ options.must_set_database_options = PROMPT_DATABASE_OPTIONS
+
#correct database
if options.dbms == 'embedded':
print "WARNING: HostName for postgres server " + options.database_host + \
@@ -4650,154 +3538,6 @@ def main():
if options.exit_message is not None:
print options.exit_message
-
-# A Python replacement for java.util.Properties
-# Based on http://code.activestate.com/recipes
-# /496795-a-python-replacement-for-javautilproperties/
-class Properties(object):
- def __init__(self, props=None):
- self._props = {}
- self._origprops = {}
- self._keymap = {}
-
- self.othercharre = re.compile(r'(?<!\\)(\s*\=)|(?<!\\)(\s*\:)')
- self.othercharre2 = re.compile(r'(\s*\=)|(\s*\:)')
- self.bspacere = re.compile(r'\\(?!\s$)')
-
- def __parse(self, lines):
- lineno = 0
- i = iter(lines)
- for line in i:
- lineno += 1
- line = line.strip()
- if not line:
- continue
- if line[0] == '#':
- continue
- escaped = False
- sepidx = -1
- flag = 0
- m = self.othercharre.search(line)
- if m:
- first, last = m.span()
- start, end = 0, first
- flag = 1
- wspacere = re.compile(r'(?<![\\\=\:])(\s)')
- else:
- if self.othercharre2.search(line):
- wspacere = re.compile(r'(?<![\\])(\s)')
- start, end = 0, len(line)
- m2 = wspacere.search(line, start, end)
- if m2:
- first, last = m2.span()
- sepidx = first
- elif m:
- first, last = m.span()
- sepidx = last - 1
- while line[-1] == '\\':
- nextline = i.next()
- nextline = nextline.strip()
- lineno += 1
- line = line[:-1] + nextline
- if sepidx != -1:
- key, value = line[:sepidx], line[sepidx + 1:]
- else:
- key, value = line, ''
- self.process_pair(key, value)
-
- def process_pair(self, key, value):
- """
- Adds or overrides the property with the given key.
- """
- oldkey = key
- oldvalue = value
- keyparts = self.bspacere.split(key)
- strippable = False
- lastpart = keyparts[-1]
- if lastpart.find('\\ ') != -1:
- keyparts[-1] = lastpart.replace('\\', '')
- elif lastpart and lastpart[-1] == ' ':
- strippable = True
- key = ''.join(keyparts)
- if strippable:
- key = key.strip()
- oldkey = oldkey.strip()
- oldvalue = self.unescape(oldvalue)
- value = self.unescape(value)
- self._props[key] = None if value is None else value.strip()
- if self._keymap.has_key(key):
- oldkey = self._keymap.get(key)
- self._origprops[oldkey] = None if oldvalue is None else oldvalue.strip()
- else:
- self._origprops[oldkey] = None if oldvalue is None else oldvalue.strip()
- self._keymap[key] = oldkey
-
- def unescape(self, value):
- newvalue = value
- if not value is None:
- newvalue = value.replace('\:', ':')
- newvalue = newvalue.replace('\=', '=')
- return newvalue
-
- def removeOldProp(self, key):
- if self._origprops.has_key(key):
- del self._origprops[key]
- pass
-
- def load(self, stream):
- if type(stream) is not file:
- raise TypeError, 'Argument should be a file object!'
- if stream.mode != 'r':
- raise ValueError, 'Stream should be opened in read-only mode!'
- try:
- self.fileName = os.path.abspath(stream.name)
- lines = stream.readlines()
- self.__parse(lines)
- except IOError:
- raise
-
- def get_property(self, key):
- return self._props.get(key, '')
-
- def propertyNames(self):
- return self._props.keys()
-
- def getPropertyDict(self):
- return self._props
-
- def __getitem__(self, name):
- return self.get_property(name)
-
- def __getattr__(self, name):
- try:
- return self.__dict__[name]
- except KeyError:
- if hasattr(self._props, name):
- return getattr(self._props, name)
-
- def store(self, out, header=""):
- """ Write the properties list to the stream 'out' along
- with the optional 'header'
- This function will attempt to close the file handler once it's done.
- """
- if out.mode[0] != 'w':
- raise ValueError, 'Steam should be opened in write mode!'
- try:
- out.write(''.join(('#', ASF_LICENSE_HEADER, '\n')))
- out.write(''.join(('#', header, '\n')))
- # Write timestamp
- tstamp = time.strftime('%a %b %d %H:%M:%S %Z %Y', time.localtime())
- out.write(''.join(('#', tstamp, '\n')))
- # Write properties from the pristine dictionary
- for prop, val in self._origprops.items():
- if val is not None:
- out.write(''.join((prop, '=', val, '\n')))
- except IOError:
- raise
- finally:
- if out:
- out.close()
-
if __name__ == "__main__":
try:
main()
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari_server/dbConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration.py b/ambari-server/src/main/python/ambari_server/dbConfiguration.py
index daa84ab..2670f7e 100644
--- a/ambari-server/src/main/python/ambari_server/dbConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/dbConfiguration.py
@@ -18,11 +18,13 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
-from ambari_commons import OSCheck
+from ambari_commons import OSConst
from ambari_commons.exceptions import FatalException
-from ambari_commons.logging_utils import print_error_msg
+from ambari_commons.logging_utils import print_error_msg, print_info_msg, print_warning_msg
+from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_server.serverConfiguration import PRESS_ENTER_MSG, get_value_from_properties
from ambari_server.setupSecurity import SECURITY_IS_ENCRYPTION_ENABLED
-from serverConfiguration import get_ambari_properties
+from ambari_server.userInput import get_validated_string_input
#Database settings
@@ -31,79 +33,72 @@ DB_STATUS_RUNNING_DEFAULT = "running"
SETUP_DB_CONNECT_TIMEOUT = 5
SETUP_DB_CONNECT_ATTEMPTS = 3
-DATABASE_INDEX = 0
USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
PASSWORD_PATTERN = "^[a-zA-Z0-9_-]*$"
DATABASE_NAMES = ["postgres", "oracle", "mysql"]
-DATABASE_STORAGE_NAMES = ["Database", "Service", "Database"]
-DATABASE_PORTS = ["5432", "1521", "3306"]
-DATABASE_DRIVER_NAMES = ["org.postgresql.Driver", "oracle.jdbc.driver.OracleDriver", "com.mysql.jdbc.Driver"]
-DATABASE_CONNECTION_STRINGS = [
- "jdbc:postgresql://{0}:{1}/{2}",
- "jdbc:oracle:thin:@{0}:{1}/{2}",
- "jdbc:mysql://{0}:{1}/{2}"]
-DATABASE_CONNECTION_STRINGS_ALT = [
- "jdbc:postgresql://{0}:{1}/{2}",
- "jdbc:oracle:thin:@{0}:{1}:{2}",
- "jdbc:mysql://{0}:{1}/{2}"]
-ORACLE_SID_PATTERN = "jdbc:oracle:thin:@.+:.+/.+"
-ORACLE_SNAME_PATTERN = "jdbc:oracle:thin:@.+:.+:.+"
-
-DATABASE_CLI_TOOLS = [["psql"], ["sqlplus", "sqlplus64"], ["mysql"]]
-DATABASE_CLI_TOOLS_DESC = ["psql", "sqlplus", "mysql"]
-DATABASE_CLI_TOOLS_USAGE = ['su -postgres --command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'"',
- 'sqlplus {1}/{2} < {0} ',
- 'mysql --user={1} --password={2} {3}<{0}']
-
-MYSQL_INIT_SCRIPT = '/var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql'
-DATABASE_INIT_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-CREATE.sql',
- '/var/lib/ambari-server/resources/Ambari-DDL-Oracle-CREATE.sql',
- MYSQL_INIT_SCRIPT]
-DATABASE_DROP_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-DROP.sql',
- '/var/lib/ambari-server/resources/Ambari-DDL-Oracle-DROP.sql',
- '/var/lib/ambari-server/resources/Ambari-DDL-MySQL-DROP.sql']
+
+AMBARI_DATABASE_NAME = "ambari"
+AMBARI_DATABASE_TITLE = "ambari"
+
+STORAGE_TYPE_LOCAL = 'local'
+STORAGE_TYPE_REMOTE = 'remote'
+
+#
+# Database configuration helper classes
+#
+class DBMSDesc:
+ def __init__(self, i_dbms_key, i_storage_key, i_dbms_name, i_storage_name, i_fn_create_config):
+ self.dbms_key = i_dbms_key
+ self.storage_key = i_storage_key
+ self.dbms_name = i_dbms_name
+ self.storage_name = i_storage_name
+ self.fn_create_config = i_fn_create_config
+
+ def create_config(self, options, properties, dbId):
+ return self.fn_create_config(options, properties, self.storage_key, dbId)
+
+class DbPropKeys:
+ def __init__(self, i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key):
+ self.dbms_key = i_dbms_key
+ self.driver_key = i_driver_key
+ self.server_key = i_server_key
+ self.port_key = i_port_key
+ self.db_name_key = i_db_name_key
+ self.db_url_key = i_db_url_key
+
+class DbAuthenticationKeys:
+ def __init__(self, i_user_name_key, i_password_key, i_password_alias, i_password_filename):
+ self.user_name_key = i_user_name_key
+ self.password_key = i_password_key
+ self.password_alias = i_password_alias
+ self.password_filename = i_password_filename
+
#
# Database configuration base class
#
class DBMSConfig(object):
- def __init__(self, options, properties):
+ def __init__(self, options, properties, storage_type):
"""
#Just load the defaults. The derived classes will be able to modify them later
"""
- self.persistence_type = 'remote'
+ self.persistence_type = storage_type
self.dbms = ""
- self.driver_name = ""
+ self.driver_class_name = ""
self.database_host = ""
self.database_port = ""
self.database_name = ""
self.database_username = ""
self.password_file = None
- self.silent = options.silent
+ self.db_title = AMBARI_DATABASE_TITLE
- isSecureProp = properties.get_property(SECURITY_IS_ENCRYPTION_ENABLED)
- self.isSecure = True if isSecureProp and isSecureProp.lower() == 'true' else False
- pass
+ self.silent = DBMSConfig._init_member_with_default(options, "silent", False)
+ self.JDBC_DRIVER_INSTALL_MSG = 'Before starting Ambari Server, you must install the JDBC driver.'
- @staticmethod
- # properties = property bag that will ultimately define the type of database. Since
- # right now in Windows we only support SQL Server, this argument is not yet used.
- # dbId = additional information, that helps distinguish between various database connections
- # (Ambari vs. Metrics is a prime example)
- def create(options, properties, dbId = "Ambari"):
- #if OSCheck.is_windows_family():
- if dbId == "Ambari":
- return SQLServerAmbariDBConfig(options, properties)
- elif dbId == "Metrics":
- return SQLServerMetricsDBConfig(options, properties)
- else:
- raise FatalException(-1, "Invalid database requested: " + str(dbId))
- #else:
- # go the classic Linux way
- #return PGConfig(properties, dbId)
- #return MySQLConfig(properties, dbId)
- #return OracleConfig(properties, dbId)
+ isSecureProp = get_value_from_properties(properties, SECURITY_IS_ENCRYPTION_ENABLED, "False")
+ self.isSecure = True if isSecureProp and isSecureProp.lower() == 'true' else False
+ pass
#
@@ -124,7 +119,7 @@ class DBMSConfig(object):
return result
def setup_database(self):
- print 'Configuring {} database...'.format(self.db_title)
+ print 'Configuring {0} database...'.format(self.db_title)
#DB setup should be done last after doing any setup.
if self._is_local_database():
@@ -134,7 +129,7 @@ class DBMSConfig(object):
pass
def reset_database(self):
- print 'Resetting {} database...'.format(self.db_title)
+ print 'Resetting {0} database...'.format(self.db_title)
if self._is_local_database():
self._reset_local_database()
@@ -142,16 +137,14 @@ class DBMSConfig(object):
self._reset_remote_database()
pass
- def ensure_jdbc_driver_installed(self, args, properties):
- result = self._is_jdbc_driver_installed(properties)
+ def ensure_jdbc_driver_installed(self, properties):
+ (result, msg) = self._prompt_jdbc_driver_install(properties)
if result == -1:
- (result, msg) = self._prompt_jdbc_driver_install(properties)
- if result == -1:
- print_error_msg(msg)
- raise FatalException(-1, msg)
+ print_error_msg(msg)
+ raise FatalException(-1, msg)
if result != 1:
- if self._install_jdbc_driver(args, properties):
+ if self._install_jdbc_driver(properties, result):
return True
return False
@@ -160,18 +153,32 @@ class DBMSConfig(object):
# Private implementation
#
+ @staticmethod
+ def _init_member_with_default(options, attr_name, default_val):
+ options_val = getattr(options, attr_name, None)
+ val = options_val if options_val is not None and options_val is not "" else default_val
+ return val
+
+ @staticmethod
+ def _init_member_with_properties(options, attr_name, properties, property_key):
+ options_val = getattr(options, attr_name, None)
+ if options_val is None or options_val is "":
+ options_val = get_value_from_properties(properties, property_key, None)
+ return options_val
+
+ @staticmethod
+ def _init_member_with_prop_default(options, attr_name, properties, property_key, default_val):
+ val = DBMSConfig._init_member_with_properties(options, attr_name, properties, property_key)
+ if val is None or val is "":
+ val = default_val
+ return val
+
#
# Checks if options determine local DB configuration
#
def _is_local_database(self):
return self.persistence_type == 'local'
- def _is_jdbc_driver_installed(self, properties):
- return 1
-
- def configure_database_password(showDefault=True):
- pass
-
def _prompt_db_properties(self):
#if WINDOWS
# prompt for SQL Server host and instance name
@@ -199,15 +206,172 @@ class DBMSConfig(object):
pass
def _prompt_jdbc_driver_install(self, properties):
- return (False, "")
+ result = self._is_jdbc_driver_installed(properties)
+ if result == -1:
+ if self.silent:
+ print_error_msg(self.JDBC_DRIVER_INSTALL_MSG)
+ else:
+ print_warning_msg(self.JDBC_DRIVER_INSTALL_MSG)
+ raw_input(PRESS_ENTER_MSG)
+ result = self._is_jdbc_driver_installed(properties)
+ return (result, self.JDBC_DRIVER_INSTALL_MSG)
+
+ def _is_jdbc_driver_installed(self, properties):
+ return 1
- def _install_jdbc_driver(self, options, properties):
+ def _install_jdbc_driver(self, properties, files_list):
return False
def ensure_dbms_is_running(self, options, properties, scmStatus=None):
pass
-if OSCheck.is_windows_family():
- from ambari_server.dbConfiguration_windows import SQLServerAmbariDBConfig, SQLServerMetricsDBConfig
-#else:
-# from ambari_server.dbConfiguration_linux import PostgreSQLConfig #and potentially MySQLConfig, OracleConfig
+
+#
+# Database configuration factory base class
+#
+class DBMSConfigFactory(object):
+ def select_dbms(self, options):
+ '''
+ # Base declaration of the DBMS selection method.
+ :return: DBMS index in the descriptor table
+ '''
+ pass
+
+ def create(self, options, properties, dbId = "Ambari"):
+ """
+ # Base declaration of the factory method. The outcome of the derived implementations
+ # is expected to be a subclass of DBMSConfig.
+ # properties = property bag that will ultimately define the type of database. Since
+ # right now in Windows we only support SQL Server, this argument is not yet used.
+ # dbId = additional information, that helps distinguish between various database connections
+ # (Ambari vs. Metrics is a prime example)
+ """
+ pass
+
+#
+# Database configuration factory for Windows
+#
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class DBMSConfigFactoryWindows(DBMSConfigFactory):
+ def __init__(self):
+ from ambari_server.dbConfiguration_windows import DATABASE_DBMS_SQLSERVER
+
+ self.DBMS_KEYS_LIST = [
+ DATABASE_DBMS_SQLSERVER
+ ]
+
+ def select_dbms(self, options):
+ # For now, we only support SQL Server in Windows, in remote mode.
+ return 0
+
+ def create(self, options, properties, dbId = "Ambari"):
+ """
+ # Windows implementation of the factory method. The outcome of the derived implementations
+ # is expected to be a subclass of DBMSConfig.
+ # properties = property bag that will ultimately define the type of database. Since
+ # right now in Windows we only support SQL Server, this argument is not yet used.
+ # dbId = additional information, that helps distinguish between various database connections
+ # (Ambari vs. Metrics is a prime example).
+ """
+ from ambari_server.dbConfiguration_windows import createSQLServerConfig
+ return createSQLServerConfig(options, properties, STORAGE_TYPE_REMOTE, dbId)
+
+ def get_supported_dbms(self):
+ return self.DBMS_KEYS_LIST
+
+#
+# Database configuration factory for Linux
+#
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class DBMSConfigFactoryLinux(DBMSConfigFactory):
+ def __init__(self):
+ from ambari_server.dbConfiguration_linux import createPGConfig, createOracleConfig, createMySQLConfig
+
+ self.DBMS_KEYS_LIST = [
+ 'postgres',
+ 'oracle',
+ 'mysql',
+ ]
+ self.DBMS_LIST = [
+ DBMSDesc(self.DBMS_KEYS_LIST[0], STORAGE_TYPE_LOCAL, 'PostgreSQL', 'Embedded', createPGConfig),
+ DBMSDesc(self.DBMS_KEYS_LIST[1], STORAGE_TYPE_REMOTE, 'Oracle', '', createOracleConfig),
+ DBMSDesc(self.DBMS_KEYS_LIST[2], STORAGE_TYPE_REMOTE, 'MySQL', '', createMySQLConfig),
+ DBMSDesc(self.DBMS_KEYS_LIST[0], STORAGE_TYPE_REMOTE, 'PostgreSQL', '', createPGConfig)
+ ]
+
+ self.DBMS_PROMPT_PATTERN = "[{0}] - {1}{2}\n"
+ self.DBMS_CHOICE_PROMPT_PATTERN = "==============================================================================\n" \
+ "Enter choice ({0}): "
+ self.JDK_VALID_CHOICES_PATTERN = "^[{0}]$"
+
+ def select_dbms(self, options):
+ try:
+ dbms_index = options.dbms_index
+ except AttributeError:
+ dbms_index = self._get_default_dbms_index(options)
+ if not options.silent:
+ n_dbms = 1
+ dbms_choice_prompt = "==============================================================================\n" \
+ "Choose one of the following options:\n"
+ dbms_choices = ''
+ for desc in self.DBMS_LIST:
+ if len(desc.storage_name) > 0:
+ dbms_storage = " ({0})".format(desc.storage_name)
+ else:
+ dbms_storage = ""
+ dbms_choice_prompt += self.DBMS_PROMPT_PATTERN.format(n_dbms, desc.dbms_name, dbms_storage)
+ dbms_choices += str(n_dbms)
+ n_dbms += 1
+
+ database_num = str(dbms_index + 1)
+ dbms_choice_prompt += self.DBMS_CHOICE_PROMPT_PATTERN.format(database_num)
+ dbms_valid_choices = self.JDK_VALID_CHOICES_PATTERN.format(dbms_choices)
+
+ database_num = get_validated_string_input(
+ dbms_choice_prompt,
+ database_num,
+ dbms_valid_choices,
+ "Invalid number.",
+ False
+ )
+
+ dbms_index = int(database_num) - 1
+ if dbms_index >= n_dbms:
+ print_info_msg('Unknown db option, default to {0} {1}.'.format(
+ self.DBMS_LIST[0].storage_name, self.DBMS_LIST[0].dbms_name))
+ dbms_index = 0
+
+ return dbms_index
+
+ def create(self, options, properties, dbId = "Ambari"):
+ """
+ # Linux implementation of the factory method. The outcome of the derived implementations
+ # is expected to be a subclass of DBMSConfig.
+ # properties = property bag that will ultimately define the type of database. Supported types are
+ # MySQL, Oracle and PostgreSQL.
+ # dbId = additional information, that helps distinguish between various database connections
+ # (Ambari vs. Metrics is a prime example).
+ """
+
+ try:
+ index = options.dbms_index
+ except AttributeError:
+ index = options.dbms_index = self._get_default_dbms_index(options)
+
+ desc = self.DBMS_LIST[index]
+ dbmsConfig = desc.create_config(options, properties, dbId)
+ return dbmsConfig
+
+ def get_supported_dbms(self):
+ return self.DBMS_KEYS_LIST
+
+ def _get_default_dbms_index(self, options):
+ try:
+ def_index = self.DBMS_KEYS_LIST.index(options.dbms)
+ except AttributeError:
+ def_index = 0
+ except ValueError:
+ def_index = 0
+
+ #TODO Distinguish between local and remote?
+ return def_index
[2/7] ambari git commit: AMBARI-8373 Refactor the OS-dependent Ambari
Server Windows components
Posted by fb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 0eae33b..64349a1 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -35,20 +35,44 @@ import shutil
from pwd import getpwnam
from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException
from ambari_server import BackupRestore
-
+
+from ambari_commons import OSCheck, OSConst, Firewall, FirewallChecks
+from ambari_commons.exceptions import *
+from ambari_commons.os_linux import *
+from ambari_commons.os_utils import *
+from ambari_commons.inet_utils import *
+
# We have to use this import HACK because the filename contains a dash
-from ambari_commons import Firewall, OSCheck, OSConst, FirewallChecks
+with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
+ with patch("os.symlink"):
+ with patch("__builtin__.open"):
+ with patch("glob.glob", return_value = ['/etc/init.d/postgresql-9.3']):
+ ambari__server = __import__('ambari-server')
with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
with patch("os.symlink"):
with patch("__builtin__.open"):
with patch("glob.glob", return_value = ['/etc/init.d/postgresql-9.3']):
- ambari_server = __import__('ambari-server')
+ from ambari_server.dbConfiguration import *
+ from ambari_server.dbConfiguration_linux import *
+ from ambari_server.properties import *
+ from ambari_server.serverConfiguration import *
+ from ambari_server.serverSetup import *
+ from ambari_server.userInput import *
-FatalException = ambari_server.FatalException
-NonFatalException = ambari_server.NonFatalException
CURR_AMBARI_VERSION = "2.0.0"
+db_YN_Inputs_test_ambariServerSetupWithCustomDbName_call_count = 0
+
+def db_YN_Inputs_ret_test_ambariServerSetupWithCustomDbName(*args, **kwargs):
+ global db_YN_Inputs_test_ambariServerSetupWithCustomDbName_call_count
+ if db_YN_Inputs_test_ambariServerSetupWithCustomDbName_call_count == 0:
+ db_YN_Inputs_test_ambariServerSetupWithCustomDbName_call_count += 1
+ return False
+ else:
+ return True
+
+
class TestAmbariServer(TestCase):
def setUp(self):
out = StringIO.StringIO()
@@ -58,27 +82,18 @@ class TestAmbariServer(TestCase):
def tearDown(self):
sys.stdout = sys.__stdout__
-
- @patch.object(ambari_server, 'configure_database_username_password')
- @patch.object(ambari_server, 'run_os_command')
- @patch('optparse.Values')
- def test_configure_pg_hba_ambaridb_users(self, OptParseValuesMock,
- run_os_command_method,
- configure_database_username_password_method):
+ @patch("ambari_server.dbConfiguration_linux.run_os_command")
+ def test_configure_pg_hba_ambaridb_users(self, run_os_command_method):
# Prepare mocks
run_os_command_method.return_value = (0, "", "")
- opvm = OptParseValuesMock.return_value
- opvm.database_username = "ffdf"
+ database_username = "ffdf"
tf1 = tempfile.NamedTemporaryFile()
- ambari_server.PG_HBA_CONF_FILE = tf1.name
# Run test
- ambari_server.configure_pg_hba_ambaridb_users()
+ PGConfig._configure_pg_hba_ambaridb_users(tf1.name, database_username)
# Check results
self.assertTrue(run_os_command_method.called)
- self.assertTrue(configure_database_username_password_method.called)
- string_expected = self.get_file_string(self
- .get_samples_dir("configure_pg_hba_ambaridb_users1"))
- string_actual = self.get_file_string(ambari_server.PG_HBA_CONF_FILE)
+ string_expected = self.get_file_string(self.get_samples_dir("configure_pg_hba_ambaridb_users1"))
+ string_actual = self.get_file_string(tf1.name)
self.assertEquals(string_expected, string_actual)
pass
@@ -86,9 +101,9 @@ class TestAmbariServer(TestCase):
def test_servicename_regex(self, raw_input_method):
''' Test to make sure the service name can contain digits '''
- ambari_server.SILENT = False
+ ambari__server.SILENT = False
raw_input_method.return_value = "OT100"
- result = ambari_server.get_validated_service_name("ambari", 1)
+ result = OracleConfig._get_validated_service_name("ambari", 1)
self.assertEqual("OT100", result, "Not accepting digits")
pass
@@ -96,31 +111,31 @@ class TestAmbariServer(TestCase):
def test_dbname_regex(self, raw_input_method):
''' Test to make sure the service name can contain digits '''
- ambari_server.SILENT = False
+ ambari__server.SILENT = False
raw_input_method.return_value = "OT100"
- result = ambari_server.get_validated_db_name("ambari")
+ result = LinuxDBMSConfig._get_validated_db_name("Database", "ambari")
self.assertEqual("OT100", result, "Not accepting digits")
def test_configure_pg_hba_postgres_user(self):
tf1 = tempfile.NamedTemporaryFile()
- ambari_server.PG_HBA_CONF_FILE = tf1.name
+ PGConfig.PG_HBA_CONF_FILE = tf1.name
- with open(ambari_server.PG_HBA_CONF_FILE, 'w') as fout:
+ with open(PGConfig.PG_HBA_CONF_FILE, 'w') as fout:
fout.write("\n")
fout.write("local all all md5\n")
fout.write("host all all 0.0.0.0/0 md5\n")
fout.write("host all all ::/0 md5\n")
- ambari_server.configure_pg_hba_postgres_user()
+ PGConfig._configure_pg_hba_postgres_user()
expected = self.get_file_string(self.get_samples_dir(
"configure_pg_hba_ambaridb_users2"))
- result = self.get_file_string(ambari_server.PG_HBA_CONF_FILE)
+ result = self.get_file_string(PGConfig.PG_HBA_CONF_FILE)
self.assertEqual(expected, result, "pg_hba_conf not processed")
- mode = oct(os.stat(ambari_server.PG_HBA_CONF_FILE)[stat.ST_MODE])
+ mode = oct(os.stat(PGConfig.PG_HBA_CONF_FILE)[stat.ST_MODE])
str_mode = str(mode)[-4:]
self.assertEqual("0644", str_mode, "Wrong file permissions")
@@ -134,7 +149,7 @@ class TestAmbariServer(TestCase):
# test first input
raw_input_method.return_value = "Y"
- result = ambari_server.get_choice_string_input(prompt, default,
+ result = ambari__server.get_choice_string_input(prompt, default,
firstChoice, secondChoice)
self.assertEquals(result, True)
raw_input_method.reset_mock()
@@ -142,7 +157,7 @@ class TestAmbariServer(TestCase):
raw_input_method.return_value = "N"
- result = ambari_server.get_choice_string_input(prompt, default,
+ result = ambari__server.get_choice_string_input(prompt, default,
firstChoice, secondChoice)
self.assertEquals(result, False)
@@ -152,7 +167,7 @@ class TestAmbariServer(TestCase):
raw_input_method.return_value = ""
- result = ambari_server.get_choice_string_input(prompt, default,
+ result = ambari__server.get_choice_string_input(prompt, default,
firstChoice, secondChoice)
self.assertEquals(result, default)
@@ -166,7 +181,7 @@ class TestAmbariServer(TestCase):
raw_input_method.side_effect = side_effect
- result = ambari_server.get_choice_string_input(prompt, default,
+ result = ambari__server.get_choice_string_input(prompt, default,
firstChoice, secondChoice)
self.assertEquals(result, True)
self.assertEquals(raw_input_method.call_count, 3)
@@ -184,11 +199,11 @@ class TestAmbariServer(TestCase):
pattern = "pattern_pp"
description = "blabla2"
# check password input
- self.assertFalse(False, ambari_server.SILENT)
+ self.assertFalse(False, ambari__server.SILENT)
is_pass = True
get_pass_method.return_value = "dfdsfdsfds"
- result = ambari_server.get_validated_string_input(prompt, default,
+ result = ambari__server.get_validated_string_input(prompt, default,
pattern, description, is_pass)
self.assertEquals(get_pass_method.return_value, result)
@@ -201,7 +216,7 @@ class TestAmbariServer(TestCase):
is_pass = False
raw_input_method.return_value = "dkf90ewuf0"
- result = ambari_server.get_validated_string_input(prompt, default,
+ result = ambari__server.get_validated_string_input(prompt, default,
pattern, description, is_pass)
self.assertEquals(raw_input_method.return_value, result)
@@ -210,12 +225,12 @@ class TestAmbariServer(TestCase):
def test_get_pass_file_path(self):
- result = ambari_server.get_pass_file_path("/etc/ambari/conf_file")
+ result = ambari__server.get_pass_file_path("/etc/ambari/conf_file")
self.assertEquals("/etc/ambari/password.dat", result)
pass
- @patch.object(ambari_server, 'setup_security')
+ @patch.object(ambari__server, 'setup_security')
@patch('optparse.OptionParser')
def test_main_test_setup_security(self, OptionParserMock,
setup_security_method):
@@ -227,46 +242,46 @@ class TestAmbariServer(TestCase):
options.sid_or_sname = "sid"
setup_security_method.return_value = None
- ambari_server.main()
+ ambari__server.main()
- ambari_server.main()
+ ambari__server.main()
self.assertTrue(setup_security_method.called)
- self.assertFalse(False, ambari_server.VERBOSE)
- self.assertFalse(False, ambari_server.SILENT)
-
- @patch.object(ambari_server, 'setup_ambari_krb5_jaas')
- @patch.object(ambari_server, 'setup_master_key')
- @patch.object(ambari_server, 'setup_component_https')
- @patch.object(ambari_server, 'setup_https')
- @patch.object(ambari_server, 'get_validated_string_input')
+ self.assertFalse(False, ambari__server.VERBOSE)
+ self.assertFalse(False, ambari__server.SILENT)
+
+ @patch.object(ambari__server, 'setup_ambari_krb5_jaas')
+ @patch.object(ambari__server, 'setup_master_key')
+ @patch.object(ambari__server, 'setup_component_https')
+ @patch.object(ambari__server, 'setup_https')
+ @patch.object(ambari__server, "get_validated_string_input")
def test_setup_security(self, get_validated_string_input_mock, setup_https,
setup_component_https, setup_master_key,
setup_ambari_krb5_jaas):
args = {}
get_validated_string_input_mock.return_value = '1'
- ambari_server.setup_security(args)
+ ambari__server.setup_security(args)
self.assertTrue(setup_https.called)
get_validated_string_input_mock.return_value = '2'
- ambari_server.setup_security(args)
+ ambari__server.setup_security(args)
self.assertTrue(setup_component_https.called)
setup_component_https.assert_called_with("Ganglia", "setup-ganglia-https",
- ambari_server.GANGLIA_HTTPS, "ganglia_cert")
+ ambari__server.GANGLIA_HTTPS, "ganglia_cert")
get_validated_string_input_mock.return_value = '3'
- ambari_server.setup_security(args)
+ ambari__server.setup_security(args)
self.assertTrue(setup_master_key.called)
get_validated_string_input_mock.return_value = '4'
- ambari_server.setup_security(args)
+ ambari__server.setup_security(args)
self.assertTrue(setup_ambari_krb5_jaas.called)
@patch('re.sub')
@patch('fileinput.FileInput')
- @patch.object(ambari_server, 'get_validated_string_input')
- @patch.object(ambari_server, 'search_file')
+ @patch("ambari_server.userInput.get_validated_string_input")
+ @patch.object(ambari__server, "search_file")
@patch('os.path.exists')
def test_setup_ambari_krb5_jaas(self, exists_mock, search_mock,
get_validated_string_input_mock,
@@ -277,7 +292,7 @@ class TestAmbariServer(TestCase):
# Negative case
try:
- ambari_server.setup_ambari_krb5_jaas()
+ ambari__server.setup_ambari_krb5_jaas()
self.fail("Should throw exception")
except NonFatalException as fe:
# Expected
@@ -292,17 +307,17 @@ class TestAmbariServer(TestCase):
fileinput_mock.return_value = [ 'keyTab=xyz', 'principal=xyz' ]
- ambari_server.setup_ambari_krb5_jaas()
+ ambari__server.setup_ambari_krb5_jaas()
self.assertTrue(fileinput_mock.called)
self.assertTrue(re_sub_mock.called)
self.assertTrue(re_sub_mock.call_args_list, [('aaa@aaa.cnn'),
('pathtokeytab')])
- @patch.object(ambari_server, 'setup')
- @patch.object(ambari_server, 'start')
- @patch.object(ambari_server, 'stop')
- @patch.object(ambari_server, 'reset')
+ @patch.object(ambari__server, 'setup')
+ @patch.object(ambari__server, 'start')
+ @patch.object(ambari__server, 'stop')
+ @patch.object(ambari__server, 'reset')
@patch('optparse.OptionParser')
def test_main_test_setup(self, OptionParserMock, reset_method, stop_method,
start_method, setup_method):
@@ -313,21 +328,21 @@ class TestAmbariServer(TestCase):
options.dbms = None
options.sid_or_sname = "sid"
- ambari_server.main()
+ ambari__server.main()
self.assertTrue(setup_method.called)
self.assertFalse(start_method.called)
self.assertFalse(stop_method.called)
self.assertFalse(reset_method.called)
- self.assertFalse(False, ambari_server.VERBOSE)
- self.assertFalse(False, ambari_server.SILENT)
+ self.assertFalse(False, ambari__server.VERBOSE)
+ self.assertFalse(False, ambari__server.SILENT)
- @patch.object(ambari_server, 'setup')
- @patch.object(ambari_server, 'start')
- @patch.object(ambari_server, 'stop')
- @patch.object(ambari_server, 'reset')
+ @patch.object(ambari__server, 'setup')
+ @patch.object(ambari__server, 'start')
+ @patch.object(ambari__server, 'stop')
+ @patch.object(ambari__server, 'reset')
@patch('optparse.OptionParser')
def test_main_test_start(self, OptionParserMock, reset_method, stop_method,
start_method, setup_method):
@@ -338,21 +353,21 @@ class TestAmbariServer(TestCase):
options.dbms = None
options.sid_or_sname = "sname"
- ambari_server.main()
+ ambari__server.main()
self.assertTrue(setup_method.called)
self.assertFalse(start_method.called)
self.assertFalse(stop_method.called)
self.assertFalse(reset_method.called)
- self.assertFalse(False, ambari_server.VERBOSE)
- self.assertFalse(False, ambari_server.SILENT)
+ self.assertFalse(False, ambari__server.VERBOSE)
+ self.assertFalse(False, ambari__server.SILENT)
- @patch.object(ambari_server, 'setup')
- @patch.object(ambari_server, 'start')
- @patch.object(ambari_server, 'stop')
- @patch.object(ambari_server, 'reset')
+ @patch.object(ambari__server, 'setup')
+ @patch.object(ambari__server, 'start')
+ @patch.object(ambari__server, 'stop')
+ @patch.object(ambari__server, 'reset')
@patch('optparse.OptionParser')
def test_main_test_start_debug_short(self, OptionParserMock, reset_method, stop_method,
start_method, setup_method):
@@ -364,20 +379,20 @@ class TestAmbariServer(TestCase):
options.dbms = None
options.sid_or_sname = "sid"
- ambari_server.main()
+ ambari__server.main()
self.assertFalse(setup_method.called)
self.assertTrue(start_method.called)
self.assertFalse(stop_method.called)
self.assertFalse(reset_method.called)
- self.assertTrue(ambari_server.SERVER_DEBUG_MODE)
+ self.assertTrue(ambari__server.SERVER_DEBUG_MODE)
- @patch.object(ambari_server, 'setup')
- @patch.object(ambari_server, 'start')
- @patch.object(ambari_server, 'stop')
- @patch.object(ambari_server, 'reset')
+ @patch.object(ambari__server, 'setup')
+ @patch.object(ambari__server, 'start')
+ @patch.object(ambari__server, 'stop')
+ @patch.object(ambari__server, 'reset')
@patch('optparse.OptionParser')
def test_main_test_start_debug_long(self, OptionParserMock, reset_method, stop_method,
start_method, setup_method):
@@ -388,21 +403,21 @@ class TestAmbariServer(TestCase):
options.dbms = None
options.sid_or_sname = "sid"
- ambari_server.main()
+ ambari__server.main()
self.assertFalse(setup_method.called)
self.assertTrue(start_method.called)
self.assertFalse(stop_method.called)
self.assertFalse(reset_method.called)
- self.assertTrue(ambari_server.SERVER_DEBUG_MODE)
+ self.assertTrue(ambari__server.SERVER_DEBUG_MODE)
- @patch.object(ambari_server, 'setup')
- @patch.object(ambari_server, 'start')
- @patch.object(ambari_server, 'stop')
- @patch.object(ambari_server, 'reset')
- @patch.object(ambari_server, 'backup')
- @patch.object(ambari_server, 'restore')
+ @patch.object(ambari__server, 'setup')
+ @patch.object(ambari__server, 'start')
+ @patch.object(ambari__server, 'stop')
+ @patch.object(ambari__server, 'reset')
+ @patch.object(ambari__server, 'backup')
+ @patch.object(ambari__server, 'restore')
@patch('optparse.OptionParser')
def test_main_test_backup(self, OptionParserMock, restore_mock, backup_mock, reset_method, stop_method,
start_method, setup_method):
@@ -413,7 +428,7 @@ class TestAmbariServer(TestCase):
options.dbms = None
options.sid_or_sname = "sname"
- ambari_server.main()
+ ambari__server.main()
self.assertTrue(backup_mock.called)
self.assertFalse(restore_mock.called)
@@ -422,15 +437,15 @@ class TestAmbariServer(TestCase):
self.assertFalse(stop_method.called)
self.assertFalse(reset_method.called)
- self.assertFalse(False, ambari_server.VERBOSE)
- self.assertFalse(False, ambari_server.SILENT)
+ self.assertFalse(False, ambari__server.VERBOSE)
+ self.assertFalse(False, ambari__server.SILENT)
- @patch.object(ambari_server, 'setup')
- @patch.object(ambari_server, 'start')
- @patch.object(ambari_server, 'stop')
- @patch.object(ambari_server, 'reset')
- @patch.object(ambari_server, 'backup')
- @patch.object(ambari_server, 'restore')
+ @patch.object(ambari__server, 'setup')
+ @patch.object(ambari__server, 'start')
+ @patch.object(ambari__server, 'stop')
+ @patch.object(ambari__server, 'reset')
+ @patch.object(ambari__server, 'backup')
+ @patch.object(ambari__server, 'restore')
@patch('optparse.OptionParser')
def test_main_test_restore(self, OptionParserMock, restore_mock, backup_mock, reset_method, stop_method,
start_method, setup_method):
@@ -441,7 +456,7 @@ class TestAmbariServer(TestCase):
options.dbms = None
options.sid_or_sname = "sname"
- ambari_server.main()
+ ambari__server.main()
self.assertTrue(restore_mock.called)
self.assertFalse(backup_mock.called)
@@ -450,13 +465,13 @@ class TestAmbariServer(TestCase):
self.assertFalse(stop_method.called)
self.assertFalse(reset_method.called)
- self.assertFalse(False, ambari_server.VERBOSE)
- self.assertFalse(False, ambari_server.SILENT)
+ self.assertFalse(False, ambari__server.VERBOSE)
+ self.assertFalse(False, ambari__server.SILENT)
- @patch.object(ambari_server, 'setup')
- @patch.object(ambari_server, 'start')
- @patch.object(ambari_server, 'stop')
- @patch.object(ambari_server, 'reset')
+ @patch.object(ambari__server, 'setup')
+ @patch.object(ambari__server, 'start')
+ @patch.object(ambari__server, 'stop')
+ @patch.object(ambari__server, 'reset')
@patch('optparse.OptionParser')
def test_main_test_stop(self, OptionParserMock, reset_method, stop_method,
start_method, setup_method):
@@ -468,21 +483,21 @@ class TestAmbariServer(TestCase):
options.dbms = None
options.sid_or_sname = "sid"
- ambari_server.main()
+ ambari__server.main()
self.assertFalse(setup_method.called)
self.assertFalse(start_method.called)
self.assertTrue(stop_method.called)
self.assertFalse(reset_method.called)
- self.assertFalse(False, ambari_server.VERBOSE)
- self.assertFalse(False, ambari_server.SILENT)
+ self.assertFalse(False, ambari__server.VERBOSE)
+ self.assertFalse(False, ambari__server.SILENT)
- @patch.object(ambari_server, 'setup')
- @patch.object(ambari_server, 'start')
- @patch.object(ambari_server, 'stop')
- @patch.object(ambari_server, 'reset')
+ @patch.object(ambari__server, 'setup')
+ @patch.object(ambari__server, 'start')
+ @patch.object(ambari__server, 'stop')
+ @patch.object(ambari__server, 'reset')
@patch('optparse.OptionParser')
def test_main_test_reset(self, OptionParserMock, reset_method, stop_method,
start_method, setup_method):
@@ -494,108 +509,130 @@ class TestAmbariServer(TestCase):
options.dbms = None
options.sid_or_sname = "sid"
- ambari_server.main()
+ ambari__server.main()
self.assertFalse(setup_method.called)
self.assertFalse(start_method.called)
self.assertFalse(stop_method.called)
self.assertTrue(reset_method.called)
- self.assertFalse(False, ambari_server.VERBOSE)
- self.assertFalse(False, ambari_server.SILENT)
-
+ self.assertFalse(False, ambari__server.VERBOSE)
+ self.assertFalse(False, ambari__server.SILENT)
def test_configure_postgresql_conf(self):
tf1 = tempfile.NamedTemporaryFile()
- ambari_server.POSTGRESQL_CONF_FILE = tf1.name
+ PGConfig.POSTGRESQL_CONF_FILE = tf1.name
- with open(ambari_server.POSTGRESQL_CONF_FILE, 'w') as f:
+ with open(PGConfig.POSTGRESQL_CONF_FILE, 'w') as f:
f.write("#listen_addresses = '127.0.0.1' #\n")
f.write("#listen_addresses = '127.0.0.1'")
- ambari_server.configure_postgresql_conf()
+ PGConfig._configure_postgresql_conf()
expected = self.get_file_string(self.get_samples_dir(
"configure_postgresql_conf1"))
- result = self.get_file_string(ambari_server.POSTGRESQL_CONF_FILE)
+ result = self.get_file_string(PGConfig.POSTGRESQL_CONF_FILE)
self.assertEqual(expected, result, "postgresql.conf not updated")
- mode = oct(os.stat(ambari_server.POSTGRESQL_CONF_FILE)[stat.ST_MODE])
+ mode = oct(os.stat(PGConfig.POSTGRESQL_CONF_FILE)[stat.ST_MODE])
str_mode = str(mode)[-4:]
self.assertEqual("0644", str_mode, "Wrong file permissions")
- @patch.object(ambari_server, "restart_postgres")
- @patch.object(ambari_server, "get_postgre_status")
- @patch.object(ambari_server, "configure_postgresql_conf")
- @patch.object(ambari_server, "configure_pg_hba_ambaridb_users")
- @patch.object(ambari_server, "configure_pg_hba_postgres_user")
- def test_configure_postgres(self, configure_pg_hba_postgres_user_mock,
- configure_pg_hba_ambaridb_users_mock,
+ @patch.object(OSCheck, "get_os_family")
+ @patch.object(OSCheck, "get_os_type")
+ @patch.object(OSCheck, "get_os_major_version")
+ @patch("ambari_server.dbConfiguration_linux.PGConfig._restart_postgres")
+ @patch("ambari_server.dbConfiguration_linux.PGConfig._get_postgre_status")
+ @patch("ambari_server.dbConfiguration_linux.PGConfig._configure_postgresql_conf")
+ @patch("ambari_server.dbConfiguration_linux.run_os_command")
+ def test_configure_postgres(self,
+ run_os_command_mock,
configure_postgresql_conf_mock,
get_postgre_status_mock,
- restart_postgres_mock):
+ restart_postgres_mock,
+ get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
+ args = MagicMock()
+ properties = Properties()
+
+ args.dbms_index = 0
+
+ del args.dbms
+ del args.database_host
+ del args.database_port
+ del args.database_name
+ del args.database_username
+ del args.database_password
+ del args.silent
+
+ factory = DBMSConfigFactory()
+ dbConfig = factory.create(args, properties)
+
+ self.assertTrue(dbConfig.dbms, "postgres")
+ self.assertTrue(dbConfig.persistence_type, "local")
+
tf1 = tempfile.NamedTemporaryFile()
tf2 = tempfile.NamedTemporaryFile()
- ambari_server.PG_HBA_CONF_FILE = tf1.name
- ambari_server.PG_HBA_CONF_FILE_BACKUP = tf2.name
- args = MagicMock()
+
+ PGConfig.PG_HBA_CONF_FILE = tf1.name
+ PGConfig.PG_HBA_CONF_FILE_BACKUP = tf2.name
out = StringIO.StringIO()
sys.stdout = out
- retcode, out1, err = ambari_server.configure_postgres()
+ retcode, out1, err = dbConfig._configure_postgres()
sys.stdout = sys.__stdout__
self.assertEqual(0, retcode)
self.assertEqual("Backup for pg_hba found, reconfiguration not required\n",
out.getvalue())
+ tf2.close()
- ambari_server.PG_HBA_CONF_FILE_BACKUP = tempfile.mktemp()
- get_postgre_status_mock.return_value = ambari_server.PG_STATUS_RUNNING, 0, "", ""
+ get_postgre_status_mock.return_value = PGConfig.PG_STATUS_RUNNING, 0, "", ""
+ run_os_command_mock.return_value = 0, "", ""
restart_postgres_mock.return_value = 0, "", ""
- rcode, out, err = ambari_server.configure_postgres()
+ rcode, out, err = dbConfig._configure_postgres()
- self.assertTrue(os.path.isfile(ambari_server.PG_HBA_CONF_FILE_BACKUP),
+ self.assertTrue(os.path.isfile(PGConfig.PG_HBA_CONF_FILE_BACKUP),
"postgresql.conf backup not created")
- self.assertTrue(configure_pg_hba_postgres_user_mock.called)
- self.assertTrue(configure_pg_hba_ambaridb_users_mock.called)
- mode = oct(os.stat(ambari_server.PG_HBA_CONF_FILE)[stat.ST_MODE])
+ self.assertTrue(run_os_command_mock.called)
+ mode = oct(os.stat(PGConfig.PG_HBA_CONF_FILE)[stat.ST_MODE])
str_mode = str(mode)[-4:]
self.assertEqual("0644", str_mode, "Wrong file permissions")
self.assertTrue(configure_postgresql_conf_mock.called)
self.assertEqual(0, rcode)
- os.unlink(ambari_server.PG_HBA_CONF_FILE_BACKUP)
+ os.unlink(PGConfig.PG_HBA_CONF_FILE_BACKUP)
+
get_postgre_status_mock.return_value = "stopped", 0, "", ""
- rcode, out, err = ambari_server.configure_postgres()
+ rcode, out, err = dbConfig._configure_postgres()
self.assertEqual(0, rcode)
- os.unlink(ambari_server.PG_HBA_CONF_FILE_BACKUP)
+ os.unlink(PGConfig.PG_HBA_CONF_FILE_BACKUP)
sys.stdout = sys.__stdout__
@patch("time.sleep")
@patch("subprocess.Popen")
- @patch.object(ambari_server, "run_os_command")
- @patch.object(ambari_server, "get_postgre_status")
- @patch.object(ambari_server, "print_info_msg")
+ @patch("ambari_server.dbConfiguration_linux.run_os_command")
+ @patch("ambari_server.dbConfiguration_linux.PGConfig._get_postgre_status")
+ @patch("ambari_server.dbConfiguration_linux.print_info_msg")
def test_restart_postgres(self, printInfoMsg_mock, get_postgre_status_mock,
run_os_command_mock, popenMock, sleepMock):
p = MagicMock()
p.poll.return_value = 0
popenMock.return_value = p
- retcode, out, err = ambari_server.restart_postgres()
+ retcode, out, err = PGConfig._restart_postgres()
self.assertEqual(0, retcode)
p.poll.return_value = None
get_postgre_status_mock.return_value = "stopped", 0, "", ""
run_os_command_mock.return_value = (1, None, None)
- retcode, out, err = ambari_server.restart_postgres()
+ retcode, out, err = PGConfig._restart_postgres()
self.assertEqual(1, retcode)
@patch("shlex.split")
@patch("subprocess.Popen")
- @patch.object(ambari_server, "print_info_msg")
+ @patch("ambari_commons.os_linux.print_info_msg")
def test_run_os_command(self, printInfoMsg_mock, popenMock, splitMock):
p = MagicMock()
@@ -605,151 +642,283 @@ class TestAmbariServer(TestCase):
# with list arg
cmd = ["exec", "arg"]
- ambari_server.run_os_command(cmd)
+ run_os_command(cmd)
self.assertFalse(splitMock.called)
# with str arg
- resp = ambari_server.run_os_command("runme")
+ resp = run_os_command("runme")
self.assertEqual(3, resp[0])
self.assertTrue(splitMock.called)
- @patch.object(ambari_server, "get_conf_dir")
- @patch.object(ambari_server, "search_file")
+ @patch.object(ambari__server, "get_conf_dir")
+ @patch.object(ambari__server, "search_file")
def test_write_property(self, search_file_mock, get_conf_dir_mock):
expected_content = "key1=val1\n"
tf1 = tempfile.NamedTemporaryFile()
search_file_mock.return_value = tf1.name
- ambari_server.write_property("key1", "val1")
+ ambari__server.write_property("key1", "val1")
result = tf1.read()
self.assertTrue(expected_content in result)
- @patch.object(ambari_server, "configure_database_username_password")
- @patch.object(ambari_server, "run_os_command")
- def test_setup_db(self, run_os_command_mock,
+ #TODO Deprecate
+ @patch.object(ambari__server, "configure_database_username_password")
+ @patch.object(ambari__server, 'run_os_command')
+ def test_setup_db_old(self, run_os_command_mock,
configure_database_username_password_mock):
run_os_command_mock.return_value = (0, None, None)
- result = ambari_server.setup_db(MagicMock())
+ result = ambari__server.setup_db(MagicMock())
self.assertTrue(configure_database_username_password_mock.called)
self.assertEqual((0, None, None), result)
- @patch.object(ambari_server, "configure_database_username_password")
+ #TODO Deprecate
+ @patch.object(ambari__server, "configure_database_username_password")
@patch("time.sleep")
- @patch.object(ambari_server, "run_os_command")
- def test_setup_db_connect_attempts_fail(self, run_os_command_mock,
+ @patch.object(ambari__server, 'run_os_command')
+ def test_setup_db_connect_attempts_fail_old(self, run_os_command_mock,
sleep_mock, config_db_mock):
run_os_command_mock.side_effect = [(1, "error", "error"), (1, "error", "error"),
(1, "error", "error")]
- result = ambari_server.setup_db(MagicMock())
+ result = ambari__server.setup_db(MagicMock())
self.assertTrue(run_os_command_mock.called)
self.assertEqual((1, 'error', 'error') , result)
self.assertEqual(2, sleep_mock.call_count)
pass
- @patch.object(ambari_server, "configure_database_username_password")
+ #TODO Deprecate
+ @patch.object(ambari__server, "configure_database_username_password")
@patch("time.sleep")
- @patch.object(ambari_server, "run_os_command")
- def test_setup_db_connect_attempts_success(self, run_os_command_mock,
- sleep_mock, config_db_mock):
+ @patch.object(ambari__server, 'run_os_command')
+ def test_setup_db_connect_attempts_success_old(self, run_os_command_mock, sleep_mock,
+ config_db_mock):
run_os_command_mock.side_effect = [(1, "error", "error"), (0, None, None),
(0, None, None)]
- result = ambari_server.setup_db(MagicMock())
+ result = ambari__server.setup_db(MagicMock())
+ self.assertTrue(run_os_command_mock.called)
+ self.assertEqual((0, None, None) , result)
+ self.assertEqual(1, sleep_mock.call_count)
+ pass
+
+
+ @patch.object(OSCheck, "get_os_family")
+ @patch.object(OSCheck, "get_os_type")
+ @patch.object(OSCheck, "get_os_major_version")
+ @patch("time.sleep")
+ @patch("ambari_server.dbConfiguration_linux.run_os_command")
+ def test_setup_db(self, run_os_command_mock, sleep_mock,
+ get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
+ args = MagicMock()
+ properties = Properties()
+
+ args.dbms_index = 0
+
+ del args.dbms
+ del args.database_host
+ del args.database_port
+ del args.database_name
+ del args.database_username
+ del args.database_password
+ del args.silent
+
+ run_os_command_mock.return_value = (0, None, None)
+
+ factory = DBMSConfigFactory()
+ dbConfig = factory.create(args, properties)
+
+ self.assertTrue(dbConfig.dbms, "postgres")
+ self.assertTrue(dbConfig.persistence_type, "local")
+
+ result = dbConfig._setup_db()
+ self.assertEqual((0, None, None), result)
+
+ @patch.object(OSCheck, "get_os_family")
+ @patch.object(OSCheck, "get_os_type")
+ @patch.object(OSCheck, "get_os_major_version")
+ @patch("time.sleep")
+ @patch("ambari_server.dbConfiguration_linux.run_os_command")
+ def test_setup_db_connect_attempts_fail(self, run_os_command_mock, sleep_mock,
+ get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
+ args = MagicMock()
+ properties = Properties()
+
+ args.dbms_index = 0
+
+ del args.dbms
+ del args.database_host
+ del args.database_port
+ del args.database_name
+ del args.database_username
+ del args.database_password
+ del args.silent
+
+ run_os_command_mock.side_effect = [(1, "error", "error"), (1, "error", "error"),
+ (1, "error", "error")]
+
+ factory = DBMSConfigFactory()
+ dbConfig = factory.create(args, properties)
+
+ self.assertTrue(dbConfig.dbms, "postgres")
+ self.assertTrue(dbConfig.persistence_type, "local")
+
+ result = dbConfig._setup_db()
+ self.assertTrue(run_os_command_mock.called)
+ self.assertEqual((1, 'error', 'error') , result)
+ self.assertEqual(2, sleep_mock.call_count)
+ pass
+
+ @patch.object(OSCheck, "get_os_family")
+ @patch.object(OSCheck, "get_os_type")
+ @patch.object(OSCheck, "get_os_major_version")
+ @patch("time.sleep")
+ @patch("ambari_server.dbConfiguration_linux.run_os_command")
+ def test_setup_db_connect_attempts_success(self, run_os_command_mock, sleep_mock,
+ get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
+ args = MagicMock()
+ properties = Properties()
+
+ args.dbms_index = 0
+
+ del args.dbms
+ del args.database_host
+ del args.database_port
+ del args.database_name
+ del args.database_username
+ del args.database_password
+ del args.silent
+
+ run_os_command_mock.side_effect = [(1, "error", "error"), (0, None, None),
+ (0, None, None)]
+
+ factory = DBMSConfigFactory()
+ dbConfig = factory.create(args, properties)
+
+ self.assertTrue(dbConfig.dbms, "postgres")
+ self.assertTrue(dbConfig.persistence_type, "local")
+
+ result = dbConfig._setup_db()
self.assertTrue(run_os_command_mock.called)
self.assertEqual((0, None, None) , result)
self.assertEqual(1, sleep_mock.call_count)
pass
- @patch.object(ambari_server, "get_YN_input")
- @patch.object(ambari_server, "run_os_command")
- def test_check_selinux(self, run_os_command_mock, getYNInput_mock):
- run_os_command_mock.return_value = (0, ambari_server.SE_STATUS_DISABLED,
+ @patch.object(OSCheck, "get_os_family")
+ @patch.object(OSCheck, "get_os_type")
+ @patch.object(OSCheck, "get_os_major_version")
+ @patch("ambari_server.serverSetup.get_YN_input")
+ @patch("ambari_server.serverSetup.run_os_command")
+ def test_disable_security_enhancements(self, run_os_command_mock, getYNInput_mock,
+ get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
+ run_os_command_mock.return_value = (0, ambari__server.SE_STATUS_DISABLED,
None)
- rcode = ambari_server.check_selinux()
+ (rcode, err) = disable_security_enhancements()
self.assertEqual(0, rcode)
getYNInput_mock.return_value = True
run_os_command_mock.return_value = (0, "enabled "
- + ambari_server.SE_MODE_ENFORCING,
+ + ambari__server.SE_MODE_ENFORCING,
None)
- rcode = ambari_server.check_selinux()
+ (rcode, err) = disable_security_enhancements()
self.assertEqual(0, rcode)
self.assertTrue(run_os_command_mock.called)
self.assertTrue(getYNInput_mock.called)
- @patch.object(ambari_server, "print_info_msg")
+ @patch("ambari_server.serverConfiguration.print_info_msg")
def test_get_ambari_jars(self, printInfoMsg_mock):
env = "/ambari/jars"
- os.environ[ambari_server.AMBARI_SERVER_LIB] = env
- result = ambari_server.get_ambari_jars()
+ os.environ[ambari__server.AMBARI_SERVER_LIB] = env
+ result = get_ambari_jars()
self.assertEqual(env, result)
- del os.environ[ambari_server.AMBARI_SERVER_LIB]
- result = ambari_server.get_ambari_jars()
+ del os.environ[ambari__server.AMBARI_SERVER_LIB]
+ result = get_ambari_jars()
self.assertEqual("/usr/lib/ambari-server", result)
self.assertTrue(printInfoMsg_mock.called)
@patch("glob.glob")
- @patch.object(ambari_server, "print_info_msg")
+ @patch("ambari_server.serverConfiguration.print_info_msg")
def test_get_share_jars(self, printInfoMsg_mock, globMock):
globMock.return_value = ["one", "two"]
expected = "one:two:one:two"
- result = ambari_server.get_share_jars()
+ result = get_share_jars()
self.assertEqual(expected, result)
globMock.return_value = []
expected = ""
- result = ambari_server.get_share_jars()
+ result = get_share_jars()
self.assertEqual(expected, result)
@patch("glob.glob")
- @patch.object(ambari_server, "print_info_msg")
- def test_get_ambari_classpath(self, printInfoMsg_mock, globMock):
+ @patch("ambari_server.serverConfiguration.get_ambari_properties")
+ @patch("ambari_server.serverConfiguration.print_info_msg")
+ def test_get_ambari_classpath(self, printInfoMsg_mock, get_ambari_properties_mock, globMock):
+ props = {}
+ props[JDBC_DRIVER_PATH_PROPERTY] = "/driver"
+ get_ambari_properties_mock.return_value = props
globMock.return_value = ["one"]
- result = ambari_server.get_ambari_classpath()
- self.assertTrue(ambari_server.get_ambari_jars() in result)
- self.assertTrue(ambari_server.get_share_jars() in result)
+ result = get_ambari_classpath()
+ self.assertTrue(get_ambari_jars() in result)
+ self.assertTrue("/driver" in result)
+ self.assertTrue(get_share_jars() in result)
globMock.return_value = []
- result = ambari_server.get_ambari_classpath()
- self.assertTrue(ambari_server.get_ambari_jars() in result)
+ props[JDBC_DRIVER_PATH_PROPERTY] = ""
+ result = get_ambari_classpath()
+ self.assertTrue(get_ambari_jars() in result)
self.assertFalse(":" in result)
- @patch.object(ambari_server, "print_info_msg")
+ @patch("glob.glob")
+ @patch("ambari_server.serverConfiguration.get_ambari_properties")
+ @patch("ambari_server.serverConfiguration.print_info_msg")
+ def test_get_full_ambari_classpath(self, printInfoMsg_mock, get_ambari_properties_mock, globMock):
+ props = {}
+ props[JDBC_DRIVER_PATH_PROPERTY] = "/driver"
+ get_ambari_properties_mock.return_value = props
+ globMock.return_value = ["one"]
+ result = get_full_ambari_classpath()
+ self.assertTrue(get_ambari_jars() in result)
+ self.assertTrue(get_share_jars() in result)
+ props[JDBC_DRIVER_PATH_PROPERTY] = "/driver dir"
+ result = get_full_ambari_classpath()
+ self.assertEqual(result[0], '"')
+ self.assertEqual(result[len(result) - 1], '"')
+
+
+ @patch("ambari_server.serverConfiguration.print_info_msg")
def test_get_conf_dir(self, printInfoMsg_mock):
env = "/dummy/ambari/conf"
- os.environ[ambari_server.AMBARI_CONF_VAR] = env
- result = ambari_server.get_conf_dir()
+ os.environ[ambari__server.AMBARI_CONF_VAR] = env
+ result = get_conf_dir()
self.assertEqual(env, result)
- del os.environ[ambari_server.AMBARI_CONF_VAR]
- result = ambari_server.get_conf_dir()
+ del os.environ[ambari__server.AMBARI_CONF_VAR]
+ result = get_conf_dir()
self.assertEqual("/etc/ambari-server/conf", result)
def test_search_file(self):
path = os.path.dirname(__file__)
- result = ambari_server.search_file(__file__, path)
+ result = search_file(__file__, path)
expected = os.path.abspath(__file__)
self.assertEqual(expected, result)
- result = ambari_server.search_file("non_existent_file", path)
+ result = search_file("non_existent_file", path)
self.assertEqual(None, result)
- @patch.object(ambari_server, "search_file")
+ @patch("ambari_server.serverConfiguration.search_file")
def test_find_properties_file(self, search_file_mock):
# Testing case when file is not found
search_file_mock.return_value = None
try:
- ambari_server.find_properties_file()
+ find_properties_file()
self.fail("File not found'")
except FatalException:
# Expected
@@ -759,71 +928,73 @@ class TestAmbariServer(TestCase):
# Testing case when file is found
value = MagicMock()
search_file_mock.return_value = value
- result = ambari_server.find_properties_file()
+ result = find_properties_file()
self.assertTrue(result is value)
- @patch.object(ambari_server, "find_properties_file")
+ @patch.object(ambari__server, "find_properties_file")
@patch("__builtin__.open")
- @patch("ambari-server.Properties")
+ @patch.object(ambari__server, "Properties")
def test_read_ambari_user(self, properties_mock, open_mock, find_properties_file_mock):
open_mock.return_value = "dummy"
find_properties_file_mock.return_value = "dummy"
# Testing with defined user
properties_mock.return_value.__getitem__.return_value = "dummy_user"
- user = ambari_server.read_ambari_user()
+ user = ambari__server.read_ambari_user()
self.assertEquals(user, "dummy_user")
# Testing with undefined user
properties_mock.return_value.__getitem__.return_value = None
- user = ambari_server.read_ambari_user()
+ user = ambari__server.read_ambari_user()
self.assertEquals(user, None)
@patch("os.path.exists")
- @patch.object(ambari_server, "set_file_permissions")
- @patch.object(ambari_server, "run_os_command")
- @patch.object(ambari_server, "get_ambari_properties")
- @patch.object(ambari_server, "get_value_from_properties")
- @patch("os.mkdir")
- def test_adjust_directory_permissions(self, mkdir_mock,
+ @patch("ambari_server.setupSecurity.set_file_permissions")
+ @patch("ambari_server.setupSecurity.run_os_command")
+ @patch("ambari_server.setupSecurity.get_ambari_properties")
+ @patch("ambari_server.setupSecurity.get_value_from_properties")
+ @patch("shutil.rmtree")
+ @patch("os.makedirs")
+ def test_adjust_directory_permissions(self, mkdir_mock, rmtree_mock,
get_value_from_properties_mock, get_ambari_properties_mock,
run_os_command_mock, set_file_permissions_mock, exists_mock):
# Testing boostrap dir wipe
properties_mock = MagicMock()
get_value_from_properties_mock.return_value = "dummy_bootstrap_dir"
- ambari_server.adjust_directory_permissions("user")
+ adjust_directory_permissions("user")
exists_mock.return_value = False
- self.assertEquals(run_os_command_mock.call_args_list[0][0][0], "rm -rf dummy_bootstrap_dir")
+ self.assertTrue(set_file_permissions_mock.called)
+ self.assertTrue(rmtree_mock.called)
self.assertTrue(mkdir_mock.called)
set_file_permissions_mock.reset_mock()
# Test recursive calls
- old_list = ambari_server.NR_ADJUST_OWNERSHIP_LIST
+ old_list = configDefaults.NR_ADJUST_OWNERSHIP_LIST
- ambari_server.NR_ADJUST_OWNERSHIP_LIST = [
+ configDefaults.NR_ADJUST_OWNERSHIP_LIST = [
( "/etc/ambari-server/conf", "755", "{0}", True ),
( "/etc/ambari-server/conf/ambari.properties", "644", "{0}", False )
]
- ambari_server.adjust_directory_permissions("user")
+ adjust_directory_permissions("user")
self.assertTrue(len(set_file_permissions_mock.call_args_list) ==
- len(ambari_server.NR_ADJUST_OWNERSHIP_LIST))
+ len(configDefaults.NR_ADJUST_OWNERSHIP_LIST))
self.assertEquals(set_file_permissions_mock.call_args_list[0][0][3], True)
self.assertEquals(set_file_permissions_mock.call_args_list[1][0][3], False)
- ambari_server.NR_ADJUST_OWNERSHIP_LIST = old_list
+ configDefaults.NR_ADJUST_OWNERSHIP_LIST = old_list
@patch("os.path.exists")
- @patch.object(ambari_server, "run_os_command")
- @patch.object(ambari_server, "print_warning_msg")
- @patch.object(ambari_server, "print_info_msg")
+ @patch("ambari_commons.os_linux.os_run_os_command")
+ @patch("ambari_commons.os_linux.print_warning_msg")
+ @patch("ambari_commons.os_utils.print_info_msg")
def test_set_file_permissions(self, print_info_msg_mock, print_warning_msg_mock,
run_os_command_mock, exists_mock):
# Testing not existent file scenario
exists_mock.return_value = False
- ambari_server.set_file_permissions("dummy-file", "dummy-mod",
+ set_file_permissions("dummy-file", "dummy-mod",
"dummy-user", False)
self.assertFalse(run_os_command_mock.called)
self.assertTrue(print_info_msg_mock.called)
@@ -834,7 +1005,7 @@ class TestAmbariServer(TestCase):
# Testing OK scenario
exists_mock.return_value = True
run_os_command_mock.side_effect = [(0, "", ""), (0, "", "")]
- ambari_server.set_file_permissions("dummy-file", "dummy-mod",
+ set_file_permissions("dummy-file", "dummy-mod",
"dummy-user", False)
self.assertTrue(len(run_os_command_mock.call_args_list) == 2)
self.assertFalse(print_warning_msg_mock.called)
@@ -844,7 +1015,7 @@ class TestAmbariServer(TestCase):
# Testing first command fail
run_os_command_mock.side_effect = [(1, "", ""), (0, "", "")]
- ambari_server.set_file_permissions("dummy-file", "dummy-mod",
+ set_file_permissions("dummy-file", "dummy-mod",
"dummy-user", False)
self.assertTrue(len(run_os_command_mock.call_args_list) == 2)
self.assertTrue(print_warning_msg_mock.called)
@@ -854,7 +1025,7 @@ class TestAmbariServer(TestCase):
# Testing second command fail
run_os_command_mock.side_effect = [(0, "", ""), (1, "", "")]
- ambari_server.set_file_permissions("dummy-file", "dummy-mod",
+ set_file_permissions("dummy-file", "dummy-mod",
"dummy-user", False)
self.assertTrue(len(run_os_command_mock.call_args_list) == 2)
self.assertTrue(print_warning_msg_mock.called)
@@ -866,7 +1037,7 @@ class TestAmbariServer(TestCase):
exists_mock.return_value = True
run_os_command_mock.side_effect = [(0, "", ""), (0, "", "")]
- ambari_server.set_file_permissions("dummy-file", "dummy-mod",
+ set_file_permissions("dummy-file", "dummy-mod",
"dummy-user", True)
self.assertTrue(len(run_os_command_mock.call_args_list) == 2)
self.assertTrue("-R" in run_os_command_mock.call_args_list[0][0][0])
@@ -880,7 +1051,7 @@ class TestAmbariServer(TestCase):
exists_mock.return_value = True
run_os_command_mock.side_effect = [(0, "", ""), (0, "", "")]
- ambari_server.set_file_permissions("dummy-file", "dummy-mod",
+ set_file_permissions("dummy-file", "dummy-mod",
"dummy-user", False)
self.assertTrue(len(run_os_command_mock.call_args_list) == 2)
self.assertFalse("-R" in run_os_command_mock.call_args_list[0][0][0])
@@ -891,18 +1062,24 @@ class TestAmbariServer(TestCase):
print_warning_msg_mock.reset_mock()
- @patch.object(ambari_server, "get_validated_string_input")
- @patch.object(ambari_server, "print_info_msg")
- @patch.object(ambari_server, "print_warning_msg")
- @patch.object(ambari_server, "run_os_command")
+ @patch.object(OSCheck, "get_os_family")
+ @patch.object(OSCheck, "get_os_type")
+ @patch.object(OSCheck, "get_os_major_version")
+ @patch("ambari_server.serverSetup.get_validated_string_input")
+ @patch("ambari_server.serverSetup.print_info_msg")
+ @patch("ambari_server.serverSetup.print_warning_msg")
+ @patch("ambari_server.serverSetup.run_os_command")
def test_create_custom_user(self, run_os_command_mock, print_warning_msg_mock,
- print_info_msg_mock, get_validated_string_input_mock):
+ print_info_msg_mock, get_validated_string_input_mock,
+ get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
user = "dummy-user"
get_validated_string_input_mock.return_value = user
+ checks = AmbariUserChecks()
+
# Testing scenario: absent user
run_os_command_mock.side_effect = [(0, "", "")]
- result = ambari_server.create_custom_user()
+ result = checks._create_custom_user()
self.assertFalse(print_warning_msg_mock.called)
self.assertEquals(result, (0, user))
@@ -912,7 +1089,7 @@ class TestAmbariServer(TestCase):
# Testing scenario: existing user
run_os_command_mock.side_effect = [(9, "", "")]
- result = ambari_server.create_custom_user()
+ result = checks._create_custom_user()
self.assertTrue("User dummy-user already exists" in str(print_info_msg_mock.call_args_list[1][0]))
self.assertEquals(result, (0, user))
@@ -922,25 +1099,29 @@ class TestAmbariServer(TestCase):
# Testing scenario: os command fail
run_os_command_mock.side_effect = [(1, "", "")]
- result = ambari_server.create_custom_user()
+ result = checks._create_custom_user()
self.assertTrue(print_warning_msg_mock.called)
self.assertEquals(result, (1, None))
- @patch.object(ambari_server, "read_ambari_user")
- @patch.object(ambari_server, "get_YN_input")
- @patch.object(ambari_server, "create_custom_user")
- @patch.object(ambari_server, "write_property")
- @patch.object(ambari_server, "adjust_directory_permissions")
- @patch.object(ambari_server, "print_error_msg")
+ @patch.object(OSCheck, "get_os_family")
+ @patch.object(OSCheck, "get_os_type")
+ @patch.object(OSCheck, "get_os_major_version")
+ @patch("ambari_server.serverSetup.read_ambari_user")
+ @patch("ambari_server.serverSetup.get_YN_input")
+ @patch("ambari_server.serverSetup.AmbariUserChecksLinux._create_custom_user")
+ @patch("ambari_server.serverSetup.write_property")
+ @patch("ambari_server.serverSetup.adjust_directory_permissions")
+ @patch("ambari_server.serverSetup.print_error_msg")
def test_check_ambari_user(self, print_error_msg_mock,
adjust_directory_permissions_mock, write_property_mock,
- create_custom_user_mock, get_YN_input_mock, read_ambari_user_mock):
+ create_custom_user_mock, get_YN_input_mock, read_ambari_user_mock,
+ get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
# Scenario: user is already defined, user does not want to reconfigure it
read_ambari_user_mock.return_value = "dummy-user"
get_YN_input_mock.return_value = False
- result = ambari_server.check_ambari_user()
+ result = check_ambari_user()
self.assertTrue(get_YN_input_mock.called)
self.assertFalse(write_property_mock.called)
self.assertFalse(create_custom_user_mock.called)
@@ -957,7 +1138,7 @@ class TestAmbariServer(TestCase):
read_ambari_user_mock.return_value = "dummy-user"
create_custom_user_mock.return_value = (0, "new-dummy-user")
get_YN_input_mock.return_value = True
- result = ambari_server.check_ambari_user()
+ result = check_ambari_user()
self.assertTrue(get_YN_input_mock.called)
self.assertTrue(write_property_mock.called)
self.assertTrue(write_property_mock.call_args[0][1] == "new-dummy-user")
@@ -976,7 +1157,7 @@ class TestAmbariServer(TestCase):
read_ambari_user_mock.return_value = "dummy-user"
create_custom_user_mock.return_value = (1, None)
get_YN_input_mock.return_value = True
- result = ambari_server.check_ambari_user()
+ result = check_ambari_user()
self.assertTrue(get_YN_input_mock.called)
self.assertTrue(create_custom_user_mock.called)
self.assertFalse(write_property_mock.called)
@@ -992,7 +1173,7 @@ class TestAmbariServer(TestCase):
read_ambari_user_mock.return_value = None
get_YN_input_mock.return_value = True
create_custom_user_mock.return_value = (0, "dummy-user")
- result = ambari_server.check_ambari_user()
+ result = check_ambari_user()
self.assertTrue(get_YN_input_mock.called)
self.assertTrue(create_custom_user_mock.called)
self.assertTrue(write_property_mock.called)
@@ -1010,7 +1191,7 @@ class TestAmbariServer(TestCase):
read_ambari_user_mock.return_value = None
get_YN_input_mock.return_value = True
create_custom_user_mock.return_value = (1, None)
- result = ambari_server.check_ambari_user()
+ result = check_ambari_user()
self.assertTrue(get_YN_input_mock.called)
self.assertTrue(create_custom_user_mock.called)
self.assertFalse(write_property_mock.called)
@@ -1027,7 +1208,7 @@ class TestAmbariServer(TestCase):
read_ambari_user_mock.return_value = None
get_YN_input_mock.return_value = True
create_custom_user_mock.return_value = (1, None)
- result = ambari_server.check_ambari_user()
+ result = check_ambari_user()
self.assertTrue(get_YN_input_mock.called)
self.assertTrue(create_custom_user_mock.called)
self.assertFalse(write_property_mock.called)
@@ -1042,7 +1223,7 @@ class TestAmbariServer(TestCase):
# Scenario: user is not defined and left to be root
read_ambari_user_mock.return_value = None
get_YN_input_mock.return_value = False
- result = ambari_server.check_ambari_user()
+ result = check_ambari_user()
self.assertTrue(get_YN_input_mock.called)
self.assertFalse(create_custom_user_mock.called)
self.assertTrue(write_property_mock.called)
@@ -1051,15 +1232,15 @@ class TestAmbariServer(TestCase):
self.assertEqual(result, 0)
- @patch.object(ambari_server, "search_file")
@patch("__builtin__.open")
- @patch.object(ambari_server, "read_ambari_user")
- @patch.object(ambari_server, "set_file_permissions")
- def test_store_password_file(self, set_file_permissions_mock,
- read_ambari_user_mock, open_mock, search_file_mock):
+ @patch.object(ambari__server, "search_file")
+ @patch.object(ambari__server, "set_file_permissions")
+ @patch.object(ambari__server, "read_ambari_user")
+ def test_store_password_file(self, read_ambari_user_mock, set_file_permissions_mock,
+ search_file_mock, open_mock):
search_file_mock.return_value = "/etc/ambari-server/conf/ambari.properties"
open_mock.return_value = MagicMock()
- ambari_server.store_password_file("password", "passfile")
+ ambari__server.store_password_file("password", "passfile")
self.assertTrue(set_file_permissions_mock.called)
@patch("subprocess.Popen")
@@ -1124,23 +1305,36 @@ class TestAmbariServer(TestCase):
self.assertFalse(firewall_obj.check_iptables())
self.assertEqual("err", firewall_obj.stderrdata)
-
- def test_dlprogress(self):
+ def test_download_progress(self):
out = StringIO.StringIO()
sys.stdout = out
- ambari_server.dlprogress("filename", 10, 2, 100)
+ download_progress("filename", 10, 2, 100)
sys.stdout = sys.__stdout__
self.assertNotEqual("", out.getvalue())
+ @patch("os.makedirs")
+ @patch("os.path.exists")
+ @patch("os.rename")
+ @patch("os.stat")
@patch("urllib2.urlopen")
@patch("__builtin__.open")
- @patch.object(ambari_server, "dlprogress")
- def test_track_jdk(self, dlprogress_mock, openMock, urlopenMock):
+ @patch("ambari_commons.inet_utils.download_progress")
+ @patch("ambari_commons.inet_utils.print_info_msg")
+ def test_track_jdk(self, print_info_msg_mock, download_progress_mock, openMock,
+ urlopenMock, os_stat_mock, os_rename_mock, os_path_exists_mock, os_makedirs_mock):
+ os_path_exists_mock.return_value = False
+
+ fsize = MagicMock()
+ fsize.st_size = 24576
+ os_stat_mock.return_value = fsize
+
+ inf = MagicMock()
+ inf.getheaders.return_value = ["bytes 0-24575/24576"]
u = MagicMock()
- u.info.return_value = {"Content-Length": "24576"}
+ u.info.return_value = inf
chunks = [None, "second", "first"]
def side_effect(*args, **kwargs):
@@ -1152,19 +1346,19 @@ class TestAmbariServer(TestCase):
f = MagicMock()
openMock.return_value = f
- ambari_server.track_jdk("base", "url", "local")
+ force_download_file("base", "url", progress_func=download_progress_mock)
self.assertEqual(0, len(chunks))
self.assertTrue(f.write.called)
- self.assertTrue(f.flush.called)
self.assertTrue(f.close.called)
- self.assertEqual(2, len(dlprogress_mock.call_args_list))
+ self.assertEqual(2, len(download_progress_mock.call_args_list))
+ self.assertTrue(print_info_msg_mock.called)
@patch("shutil.copy")
@patch("os.path.join")
@patch("os.path.exists")
@patch("os.path.isdir")
- @patch.object(ambari_server, "get_ambari_properties")
+ @patch.object(ambari__server, "get_ambari_properties")
def test_install_jce_manualy(self, get_ambari_properties_mock, \
os_path_isdir_mock, os_path_exists_mock, os_path_join_mock, \
shutil_copy_mock):
@@ -1178,7 +1372,7 @@ class TestAmbariServer(TestCase):
# Case when JCE path doesn't exists
os_path_exists_mock.return_value = False
try:
- ambari_server.install_jce_manualy(args)
+ ambari__server.install_jce_manualy(args)
self.fail("Should throw exception")
except FatalException as fe:
# Expected
@@ -1190,7 +1384,7 @@ class TestAmbariServer(TestCase):
os_path_exists_mock.return_value = True
os_path_isdir_mock.return_value = True
try:
- ambari_server.install_jce_manualy(args)
+ ambari__server.install_jce_manualy(args)
self.fail("Should throw exception")
except FatalException as fe:
# Expected
@@ -1201,12 +1395,12 @@ class TestAmbariServer(TestCase):
os_path_isdir_mock.return_value = False
os_path_join_mock.return_value = \
"/var/lib/ambari-server/resources/jce_policy-6.zip"
- ambari_server.install_jce_manualy(args)
+ ambari__server.install_jce_manualy(args)
self.assertTrue(shutil_copy_mock.called)
shutil_copy_mock.side_effect = Exception("exception")
try:
- ambari_server.install_jce_manualy(args)
+ ambari__server.install_jce_manualy(args)
self.fail("Should throw exception because of not found jce_policy-6.zip")
except Exception:
# Expected
@@ -1215,20 +1409,20 @@ class TestAmbariServer(TestCase):
shutil_copy_mock.side_effect = None
args.jce_policy = None
- ambari_server.install_jce_manualy(args)
+ ambari__server.install_jce_manualy(args)
- @patch.object(ambari_server, "get_validated_filepath_input")
- @patch.object(ambari_server, "run_os_command")
- @patch.object(ambari_server, "get_truststore_type")
+ @patch.object(ambari__server, "get_validated_filepath_input")
+ @patch.object(ambari__server, 'run_os_command')
+ @patch.object(ambari__server, "get_truststore_type")
@patch("__builtin__.open")
- @patch.object(ambari_server, "find_properties_file")
- @patch.object(ambari_server, "run_component_https_cmd")
- @patch.object(ambari_server, "get_delete_cert_command")
- @patch.object(ambari_server, "get_truststore_password")
- @patch.object(ambari_server, "get_truststore_path")
- @patch.object(ambari_server, "get_YN_input")
- @patch.object(ambari_server, "get_ambari_properties")
- @patch.object(ambari_server, "find_jdk")
+ @patch.object(ambari__server, "find_properties_file")
+ @patch.object(ambari__server, "run_component_https_cmd")
+ @patch.object(ambari__server, "get_delete_cert_command")
+ @patch.object(ambari__server, "get_truststore_password")
+ @patch.object(ambari__server, "get_truststore_path")
+ @patch.object(ambari__server, "get_YN_input")
+ @patch.object(ambari__server, "get_ambari_properties")
+ @patch.object(ambari__server, "find_jdk")
def test_setup_component_https(self, find_jdk_mock, get_ambari_properties_mock, get_YN_input_mock,
get_truststore_path_mock, get_truststore_password_mock,
get_delete_cert_command_mock, run_component_https_cmd_mock,
@@ -1242,18 +1436,18 @@ class TestAmbariServer(TestCase):
property = "use_ssl"
alias = "alias"
#Silent mode
- ambari_server.SILENT = True
- ambari_server.setup_component_https(component, command, property, alias)
+ ambari__server.SILENT = True
+ ambari__server.setup_component_https(component, command, property, alias)
self.assertEqual('command is not enabled in silent mode.\n', out.getvalue())
sys.stdout = sys.__stdout__
#Verbouse mode and jdk_path is None
- ambari_server.SILENT = False
+ ambari__server.SILENT = False
p = get_ambari_properties_mock.return_value
# Use ssl
p.get_property.side_effect = ["true"]
# Dont disable ssl
get_YN_input_mock.side_effect = [False]
- ambari_server.setup_component_https(component, command, property, alias)
+ ambari__server.setup_component_https(component, command, property, alias)
self.assertTrue(p.get_property.called)
self.assertTrue(get_YN_input_mock.called)
p.get_property.reset_mock()
@@ -1262,7 +1456,7 @@ class TestAmbariServer(TestCase):
p.get_property.side_effect = ["false"]
# Dont enable ssl
get_YN_input_mock.side_effect = [False]
- ambari_server.setup_component_https(component, command, property, alias)
+ ambari__server.setup_component_https(component, command, property, alias)
self.assertTrue(p.get_property.called)
self.assertTrue(get_YN_input_mock.called)
p.get_property.reset_mock()
@@ -1270,7 +1464,7 @@ class TestAmbariServer(TestCase):
# Cant find jdk
find_jdk_mock.return_value = None
try:
- ambari_server.setup_component_https(component, command, property, alias)
+ ambari__server.setup_component_https(component, command, property, alias)
self.fail("Should throw exception")
except FatalException as fe:
# Expected
@@ -1284,7 +1478,7 @@ class TestAmbariServer(TestCase):
get_truststore_path_mock.return_value = "/truststore_path"
get_truststore_password_mock.return_value = "/truststore_password"
get_delete_cert_command_mock.return_value = "rm -f"
- ambari_server.setup_component_https(component, command, property, alias)
+ ambari__server.setup_component_https(component, command, property, alias)
self.assertTrue(p.process_pair.called)
self.assertTrue(get_truststore_path_mock.called)
@@ -1305,7 +1499,7 @@ class TestAmbariServer(TestCase):
#Verbouse mode and jdk_path is not None (use_https = false) and import cert
p.get_property.side_effect = ["false"]
get_YN_input_mock.side_effect = [True]
- ambari_server.setup_component_https(component, command, property, alias)
+ ambari__server.setup_component_https(component, command, property, alias)
self.assertTrue(p.process_pair.called)
self.assertTrue(get_truststore_type_mock.called)
@@ -1330,19 +1524,19 @@ class TestAmbariServer(TestCase):
run_os_command_mock.reset_mock()
get_validated_filepath_input_mock.reset_mock()
- @patch.object(ambari_server, 'adjust_directory_permissions')
- @patch.object(ambari_server, 'read_ambari_user')
- @patch.object(ambari_server, "get_validated_string_input")
- @patch.object(ambari_server, "find_properties_file")
- @patch.object(ambari_server, "get_ambari_properties")
- @patch.object(ambari_server, "is_server_runing")
- @patch.object(ambari_server, "import_cert_and_key_action")
- @patch.object(ambari_server, "get_YN_input")
+ @patch.object(ambari__server, "adjust_directory_permissions")
+ @patch.object(ambari__server, "read_ambari_user")
+ @patch.object(ambari__server, "get_validated_string_input")
+ @patch.object(ambari__server, "find_properties_file")
+ @patch.object(ambari__server, "get_ambari_properties")
+ @patch.object(ambari__server, "is_server_runing")
+ @patch.object(ambari__server, "import_cert_and_key_action")
+ @patch.object(ambari__server, "get_YN_input")
@patch("__builtin__.open")
- @patch("ambari-server.Properties")
- @patch.object(ambari_server, "is_root")
- @patch.object(ambari_server, "is_valid_cert_host")
- @patch.object(ambari_server, "is_valid_cert_exp")
+ @patch("ambari_server.properties.Properties")
+ @patch.object(ambari__server, "is_root")
+ @patch.object(ambari__server, "is_valid_cert_host")
+ @patch.object(ambari__server, "is_valid_cert_exp")
def test_setup_https(self, is_valid_cert_exp_mock, is_valid_cert_host_mock, \
is_root_mock, Properties_mock, open_Mock, get_YN_input_mock, \
import_cert_and_key_action_mock,
@@ -1360,7 +1554,7 @@ class TestAmbariServer(TestCase):
# Testing call under non-root
is_root_mock.return_value = False
try:
- ambari_server.setup_https(args)
+ ambari__server.setup_https(args)
self.fail("Should throw exception")
except FatalException as fe:
# Expected
@@ -1379,8 +1573,8 @@ class TestAmbariServer(TestCase):
" call('client.api.ssl.port'),\n" + \
" call('client.api.ssl.port'),\n call('api.ssl')]"
process_pair_expected = "[call('client.api.ssl.port', '4444')]"
- ambari_server.SILENT = False
- ambari_server.setup_https(args)
+ ambari__server.SILENT = False
+ ambari__server.setup_https(args)
self.assertTrue(p.process_pair.called)
self.assertTrue(p.get_property.call_count == 4)
@@ -1401,7 +1595,7 @@ class TestAmbariServer(TestCase):
get_property_expected = "[call('security.server.keys_dir'),\n" + \
" call('client.api.ssl.port'),\n call('api.ssl')]"
process_pair_expected = "[call('api.ssl', 'false')]"
- ambari_server.setup_https(args)
+ ambari__server.setup_https(args)
self.assertTrue(p.process_pair.called)
self.assertTrue(p.get_property.call_count == 3)
@@ -1423,7 +1617,7 @@ class TestAmbariServer(TestCase):
get_property_expected = "[call('security.server.keys_dir'),\n" + \
" call('client.api.ssl.port'),\n call('api.ssl')]"
process_pair_expected = "[call('client.api.ssl.port', '4444')]"
- ambari_server.setup_https(args)
+ ambari__server.setup_https(args)
self.assertTrue(p.process_pair.called)
self.assertTrue(p.get_property.call_count == 3)
@@ -1445,7 +1639,7 @@ class TestAmbariServer(TestCase):
get_property_expected = "[call('security.server.keys_dir'),\n" + \
" call('client.api.ssl.port'),\n call('api.ssl')]"
process_pair_expected = "[]"
- ambari_server.setup_https(args)
+ ambari__server.setup_https(args)
self.assertFalse(p.process_pair.called)
self.assertTrue(p.get_property.call_count == 3)
@@ -1467,7 +1661,7 @@ class TestAmbariServer(TestCase):
get_property_expected = "[call('security.server.keys_dir'),\n" + \
" call('client.api.ssl.port'),\n call('api.ssl')]"
process_pair_expected = "[call('client.api.ssl.port', '4444')]"
- self.assertFalse(ambari_server.setup_https(args))
+ self.assertFalse(ambari__server.setup_https(args))
self.assertTrue(p.process_pair.called)
self.assertTrue(p.get_property.call_count == 3)
self.assertEqual(str(p.get_property.call_args_list), get_property_expected)
@@ -1481,9 +1675,9 @@ class TestAmbariServer(TestCase):
import_cert_and_key_action_mock.reset_mock()
#Case #6: if silent mode is enabled
- ambari_server.SILENT = True
+ ambari__server.SILENT = True
try:
- ambari_server.setup_https(args)
+ ambari__server.setup_https(args)
self.fail("Should throw exception")
except NonFatalException as fe:
self.assertTrue("setup-https is not enabled in silent mode" in fe.reason)
@@ -1494,18 +1688,18 @@ class TestAmbariServer(TestCase):
import_cert_and_key_action_mock.reset_mock()
#Case #7: read property throw exception
- ambari_server.SILENT = False
+ ambari__server.SILENT = False
find_properties_file_mock.return_value = "propertyFile"
p.get_property.side_effect = KeyError("Failed to read property")
try:
- ambari_server.setup_https(args)
+ ambari__server.setup_https(args)
self.fail("Should throw exception")
except FatalException as fe:
self.assertTrue("Failed to read property" in fe.reason)
- @patch.object(ambari_server, "import_cert_and_key")
+ @patch.object(ambari__server, "import_cert_and_key")
def test_import_cert_and_key_action(self, import_cert_and_key_mock):
import_cert_and_key_mock.return_value = True
properties = MagicMock()
@@ -1514,24 +1708,24 @@ class TestAmbariServer(TestCase):
expect_process_pair = "[call('client.api.ssl.cert_name', 'https.crt'),\n" + \
" call('client.api.ssl.key_name', 'https.key'),\n" + \
" call('api.ssl', 'true')]"
- ambari_server.import_cert_and_key_action("key_dir", properties)
+ ambari__server.import_cert_and_key_action("key_dir", properties)
self.assertEqual(str(properties.process_pair.call_args_list), \
expect_process_pair)
- @patch.object(ambari_server, "remove_file")
- @patch.object(ambari_server, "copy_file")
- @patch.object(ambari_server, "read_ambari_user")
- @patch.object(ambari_server, "set_file_permissions")
- @patch.object(ambari_server, "import_file_to_keystore")
+ @patch.object(ambari__server, "remove_file")
+ @patch.object(ambari__server, "copy_file")
+ @patch.object(ambari__server, "read_ambari_user")
+ @patch.object(ambari__server, "set_file_permissions")
+ @patch.object(ambari__server, "import_file_to_keystore")
@patch("__builtin__.open")
- @patch.object(ambari_server, "run_os_command")
+ @patch.object(ambari__server, 'run_os_command')
@patch("os.path.join")
@patch("os.path.isfile")
@patch('__builtin__.raw_input')
- @patch.object(ambari_server, "get_validated_string_input")
- @patch.object(ambari_server, "is_valid_cert_host")
- @patch.object(ambari_server, "is_valid_cert_exp")
+ @patch.object(ambari__server, "get_validated_string_input")
+ @patch.object(ambari__server, "is_valid_cert_host")
+ @patch.object(ambari__server, "is_valid_cert_exp")
def test_import_cert_and_key(self, is_valid_cert_exp_mock, \
is_valid_cert_host_mock, \
get_validated_string_input_mock, \
@@ -1563,7 +1757,7 @@ class TestAmbariServer(TestCase):
" call('key_file_path'," + \
" 'keystore_cert_key_file_path')]"
- ambari_server.import_cert_and_key("key_dir")
+ ambari__server.import_cert_and_key("key_dir")
self.assertTrue(raw_input_mock.call_count == 2)
self.assertTrue(get_validated_string_input_mock.called)
self.assertEqual(os_path_join_mock.call_count, 8)
@@ -1571,19 +1765,19 @@ class TestAmbariServer(TestCase):
self.assertEqual(str(import_file_to_keystore_mock.call_args_list), \
expect_import_file_to_keystore)
- @patch.object(ambari_server, "remove_file")
- @patch.object(ambari_server, "copy_file")
- @patch.object(ambari_server, "generate_random_string")
- @patch.object(ambari_server, "read_ambari_user")
- @patch.object(ambari_server, "set_file_permissions")
- @patch.object(ambari_server, "import_file_to_keystore")
+ @patch.object(ambari__server, "remove_file")
+ @patch.object(ambari__server, "copy_file")
+ @patch.object(ambari__server, "generate_random_string")
+ @patch.object(ambari__server, "read_ambari_user")
+ @patch.object(ambari__server, "set_file_permissions")
+ @patch.object(ambari__server, "import_file_to_keystore")
@patch("__builtin__.open")
- @patch.object(ambari_server, "run_os_command")
+ @patch.object(ambari__server, 'run_os_command')
@patch("os.path.join")
- @patch.object(ambari_server, "get_validated_filepath_input")
- @patch.object(ambari_server, "get_validated_string_input")
- @patch.object(ambari_server, "is_valid_cert_host")
- @patch.object(ambari_server, "is_valid_cert_exp")
+ @patch.object(ambari__server, "get_validated_filepath_input")
+ @patch.object(ambari__server, "get_validated_string_input")
+ @patch.object(ambari__server, "is_valid_cert_host")
+ @patch.object(ambari__server, "is_valid_cert_exp")
def test_import_cert_and_key_with_empty_password(self, \
is_valid_cert_exp_mock, is_valid_cert_host_mock,
get_validated_string_input_mock, get_validated_filepath_input_mock, \
@@ -1613,7 +1807,7 @@ class TestAmbariServer(TestCase):
" call('key_file_path.secured'," + \
" 'keystore_cert_key_file_path')]"
- ambari_server.import_cert_and_key("key_dir")
+ ambari__server.import_cert_and_key("key_dir")
self.assertEquals(get_validated_filepath_input_mock.call_count, 2)
self.assertTrue(get_validated_string_input_mock.called)
self.assertEquals(os_path_join_mock.call_count, 8)
@@ -1623,15 +1817,15 @@ class TestAmbariServer(TestCase):
self.assertTrue(generate_random_string_mock.called)
@patch("__builtin__.open")
- @patch.object(ambari_server, "copy_file")
- @patch.object(ambari_server, "is_root")
- @patch.object(ambari_server, "read_ambari_user")
- @patch.object(ambari_server, "set_file_permissions")
- @patch.object(ambari_server, "import_file_to_keystore")
- @patch.object(ambari_server, "run_os_command")
+ @patch.object(ambari__server, "copy_file")
+ @patch.object(ambari__server, "is_root")
+ @patch.object(ambari__server, "read_ambari_user")
+ @patch.object(ambari__server, "set_file_permissions")
+ @patch.object(ambari__server, "import_file_to_keystore")
+ @patch.object(ambari__server, 'run_os_command')
@patch("os.path.join")
- @patch.object(ambari_server, "get_validated_filepath_input")
- @patch.object(ambari_server, "get_validated_string_input")
+ @patch.object(ambari__server, "get_validated_filepath_input")
+ @patch.object(ambari__server, "get_validated_string_input")
def test_import_cert_and_key_with_incorrect_password(self,
get_validated_string_input_mock, \
get_validated_filepath_input_mock, \
@@ -1654,110 +1848,110 @@ class TestAmbariServer(TestCase):
#provided password doesn't match, openssl command returns an error
run_os_command_mock.return_value = (1, "", "Some error message")
- self.assertFalse(ambari_server.import_cert_and_key_action(*["key_dir", None]))
- self.assertFalse(ambari_server.import_cert_and_key("key_dir"))
+ self.assertFalse(ambari__server.import_cert_and_key_action(*["key_dir", None]))
+ self.assertFalse(ambari__server.import_cert_and_key("key_dir"))
def test_is_valid_cert_exp(self):
#No data in certInfo
certInfo = {}
- is_valid = ambari_server.is_valid_cert_exp(certInfo)
+ is_valid = ambari__server.is_valid_cert_exp(certInfo)
self.assertFalse(is_valid)
#Issued in future
- issuedOn = (datetime.datetime.now() + datetime.timedelta(hours=1000)).strftime(ambari_server.SSL_DATE_FORMAT)
- expiresOn = (datetime.datetime.now() + datetime.timedelta(hours=2000)).strftime(ambari_server.SSL_DATE_FORMAT)
- certInfo = {ambari_server.NOT_BEFORE_ATTR: issuedOn,
- ambari_server.NOT_AFTER_ATTR: expiresOn}
- is_valid = ambari_server.is_valid_cert_exp(certInfo)
+ issuedOn = (datetime.datetime.now() + datetime.timedelta(hours=1000)).strftime(ambari__server.SSL_DATE_FORMAT)
+ expiresOn = (datetime.datetime.now() + datetime.timedelta(hours=2000)).strftime(ambari__server.SSL_DATE_FORMAT)
+ certInfo = {ambari__server.NOT_BEFORE_ATTR: issuedOn,
+ ambari__server.NOT_AFTER_ATTR: expiresOn}
+ is_valid = ambari__server.is_valid_cert_exp(certInfo)
self.assertFalse(is_valid)
#Was expired
- issuedOn = (datetime.datetime.now() - datetime.timedelta(hours=2000)).strftime(ambari_server.SSL_DATE_FORMAT)
- expiresOn = (datetime.datetime.now() - datetime.timedelta(hours=1000)).strftime(ambari_server.SSL_DATE_FORMAT)
- certInfo = {ambari_server.NOT_BEFORE_ATTR: issuedOn,
- ambari_server.NOT_AFTER_ATTR: expiresOn}
- is_valid = ambari_server.is_valid_cert_exp(certInfo)
+ issuedOn = (datetime.datetime.now() - datetime.timedelta(hours=2000)).strftime(ambari__server.SSL_DATE_FORMAT)
+ expiresOn = (datetime.datetime.now() - datetime.timedelta(hours=1000)).strftime(ambari__server.SSL_DATE_FORMAT)
+ certInfo = {ambari__server.NOT_BEFORE_ATTR: issuedOn,
+ ambari__server.NOT_AFTER_ATTR: expiresOn}
+ is_valid = ambari__server.is_valid_cert_exp(certInfo)
self.assertFalse(is_valid)
#Valid
- issuedOn = (datetime.datetime.now() - datetime.timedelta(hours=2000)).strftime(ambari_server.SSL_DATE_FORMAT)
- expiresOn = (datetime.datetime.now() + datetime.timedelta(hours=1000)).strftime(ambari_server.SSL_DATE_FORMAT)
- certInfo = {ambari_server.NOT_BEFORE_ATTR: issuedOn,
- ambari_server.NOT_AFTER_ATTR: expiresOn}
- is_valid = ambari_server.is_valid_cert_exp(certInfo)
+ issuedOn = (datetime.datetime.now() - datetime.timedelta(hours=2000)).strftime(ambari__server.SSL_DATE_FORMAT)
+ expiresOn = (datetime.datetime.now() + datetime.timedelta(hours=1000)).strftime(ambari__server.SSL_DATE_FORMAT)
+ certInfo = {ambari__server.NOT_BEFORE_ATTR: issuedOn,
+ ambari__server.NOT_AFTER_ATTR: expiresOn}
+ is_valid = ambari__server.is_valid_cert_exp(certInfo)
self.assertTrue(is_valid)
- @patch.object(ambari_server, "get_fqdn")
+ @patch.object(ambari__server, "get_fqdn")
def test_is_valid_cert_host(self, get_fqdn_mock):
#No data in certInfo
certInfo = {}
- is_valid = ambari_server.is_valid_cert_host(certInfo)
+ is_valid = ambari__server.is_valid_cert_host(certInfo)
self.assertFalse(is_valid)
#Failed to get FQDN
get_fqdn_mock.return_value = None
- is_valid = ambari_server.is_valid_cert_host(certInfo)
+ is_valid = ambari__server.is_valid_cert_host(certInfo)
self.assertFalse(is_valid)
#FQDN and Common name in certificated don't correspond
get_fqdn_mock.return_value = 'host1'
- certInfo = {ambari_server.COMMON_NAME_ATTR: 'host2'}
- is_valid = ambari_server.is_valid_cert_host(certInfo)
+ certInfo = {ambari__server.COMMON_NAME_ATTR: 'host2'}
+ is_valid = ambari__server.is_valid_cert_host(certInfo)
self.assertFalse(is_valid)
#FQDN and Common name in certificated correspond
get_fqdn_mock.return_value = 'host1'
- certInfo = {ambari_server.COMMON_NAME_ATTR: 'host1'}
- is_valid = ambari_server.is_valid_cert_host(certInfo)
+ certInfo = {ambari__server.COMMON_NAME_ATTR: 'host1'}
+ is_valid = ambari__server.is_valid_cert_host(certInfo)
self.assertTrue(is_valid)
- @patch.object(ambari_server, "get_ambari_properties")
+ @patch.object(ambari__server, "get_ambari_properties")
def test_is_valid_https_port(self, get_ambari_properties_mock):
#No ambari.properties
get_ambari_properties_mock.return_value = -1
- is_valid = ambari_server.is_valid_https_port(1111)
+ is_valid = ambari__server.is_valid_https_port(1111)
self.assertEqual(is_valid, False)
#User entered port used by one way auth
portOneWay = "1111"
portTwoWay = "2222"
validPort = "3333"
- get_ambari_properties_mock.return_value = {ambari_server.SRVR_ONE_WAY_SSL_PORT_PROPERTY: portOneWay,
- ambari_server.SRVR_TWO_WAY_SSL_PORT_PROPERTY: portTwoWay}
- is_valid = ambari_server.is_valid_https_port(portOneWay)
+ get_ambari_properties_mock.return_value = {ambari__server.SRVR_ONE_WAY_SSL_PORT_PROPERTY: portOneWay,
+ ambari__server.SRVR_TWO_WAY_SSL_PORT_PROPERTY: portTwoWay}
+ is_valid = ambari__server.is_valid_https_port(portOneWay)
self.assertEqual(is_valid, False)
#User entered port used by two way auth
- is_valid = ambari_server.is_valid_https_port(portTwoWay)
+ is_valid = ambari__server.is_valid_https_port(portTwoWay)
self.assertEqual(is_valid, False)
#User entered valid port
- get_ambari_properties_mock.return_value = {ambari_server.SRVR_ONE_WAY_SSL_PORT_PROPERTY: portOneWay,
- ambari_server.SRVR_TWO_WAY_SSL_PORT_PROPERTY: portTwoWay}
- is_valid = ambari_server.is_valid_https_port(validPort)
+ get_ambari_properties_mock.return_value = {ambari__server.SRVR_ONE_WAY_SSL_PORT_PROPERTY: portOneWay,
+ ambari__server.SRVR_TWO_WAY_SSL_PORT_PROPERTY: portTwoWay}
+ is_valid = ambari__server.is_valid_https_port(validPort)
self.assertEqual(is_valid, True)
@patch("socket.getfqdn")
@patch("urllib2.urlopen")
- @patch.object(ambari_server, "get_ambari_properties")
+ @patch.object(ambari__server, "get_ambari_properties")
def test_get_fqdn(self, get_ambari_properties_mock, url_open_mock, getfqdn_mock):
#No ambari.properties
get_ambari_properties_mock.return_value = -1
- fqdn = ambari_server.get_fqdn()
+ fqdn = ambari__server.get_fqdn()
self.assertEqual(fqdn, None)
#Check mbari_server.GET_FQDN_SERVICE_URL property name (AMBARI-2612)
#property name should be server.fqdn.service.url
- self.assertEqual(ambari_server.GET_FQDN_SERVICE_URL, "server.fqdn.service.url")
+ self.assertEqual(ambari__server.GET_FQDN_SERVICE_URL, "server.fqdn.service.url")
#Read FQDN from service
p = MagicMock()
- p[ambari_server.GET_FQDN_SERVICE_URL] = 'someurl'
+ p[ambari__server.GET_FQDN_SERVICE_URL] = 'someurl'
get_ambari_properties_mock.return_value = p
u = MagicMock()
@@ -1765,57 +1959,57 @@ class TestAmbariServer(TestCase):
u.read.return_value = host
url_open_mock.return_value = u
- fqdn = ambari_server.get_fqdn()
+ fqdn = ambari__server.get_fqdn()
self.assertEqual(fqdn, host)
#Failed to read FQDN from service, getting from socket
u.reset_mock()
u.side_effect = Exception("Failed to read FQDN from service")
getfqdn_mock.return_value = host
- fqdn = ambari_server.get_fqdn()
+ fqdn = ambari__server.get_fqdn()
self.assertEqual(fqdn, host)
- @patch.object(ambari_server, "find_properties_file")
+ @patch.object(ambari__server, "find_properties_file")
def test_get_ulimit_open_files(self, find_properties_file_mock):
# 1 - No ambari.properties
find_properties_file_mock.return_value = None
- open_files = ambari_server.get_fqdn()
+ open_files = ambari__server.get_fqdn()
self.assertEqual(open_files, None)
# 2 - With ambari.properties - ok
tf1 = tempfile.NamedTemporaryFile()
prop_value = 65000
with open(tf1.name, 'w') as fout:
- fout.write(ambari_server.ULIMIT_OPEN_FILES_KEY + '=' + str(prop_value))
+ fout.write(ambari__server.ULIMIT_OPEN_FILES_KEY + '=' + str(prop_value))
fout.close()
find_properties_file_mock.return_value = tf1.name
- open_files = ambari_server.get_ulimit_open_files()
+ open_files = ambari__server.get_ulimit_open_files()
self.assertEqual(open_files, 65000)
# 2 - With ambari.properties - default
tf1 = tempfile.NamedTemporaryFile()
prop_value = 0
with open(tf1.name, 'w') as fout:
- fout.write(ambari_server.ULIMIT_OPEN_FILES_KEY + '=' + str(prop_value))
+ fout.write(ambari__server.ULIMIT_OPEN_FILES_KEY + '=' + str(prop_value))
fout.close()
find_properties_file_mock.return_value = tf1.name
- open_files = ambari_server.get_ulimit_open_files()
- self.assertEqual(open_files, ambari_server.ULIMIT_OPEN_FILES_DEFAULT)
+ open_files = ambari__server.get_ulimit_open_files()
+ self.assertEqual(open_files, ambari__server.ULIMIT_OPEN_FILES_DEFAULT)
- @patch.object(ambari_server, "run_os_command")
+ @patch.object(ambari__server, 'run_os_command')
def test_get_cert_info(self, run_os_command_mock):
# Error running openssl command
path = 'path/to/certificate'
run_os_command_mock.return_value = -1, None, None
- cert_info = ambari_server.get_cert_info(path)
+ cert_info = ambari__server.get_cert_info(path)
self.assertEqual(cert_info, None)
#Empty result of openssl command
run_os_command_mock.return_value = 0, None, None
- cert_info = ambari_server.get_cert_info(path)
+ cert_info = ambari__server.get_cert_info(path)
self.assertEqual(cert_info, None)
#Positive scenario
@@ -1843,7 +2037,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
"""
out = out_pattern.format(notAfter=notAfter, notBefore=notBefore, subject=subject)
run_os_command_mock.return_value = 0, out, None
- cert_info = ambari_server.get_cert_info(path)
+ cert_info = ambari__server.get_cert_info(path)
self.assertEqual(cert_info['notAfter'], notAfter)
self.assertEqual(cert_info['notBefore'], notBefore)
self.assertEqual(cert_info['subject'], subject)
@@ -1862,19 +2056,19 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
inputed_value1 = 'val1'
inputed_value2 = 'val2'
raw_input_mock.return_value = inputed_value1
- input = ambari_server.get_validated_string_input(prompt, default_value, None,
+ input = ambari__server.get_validated_string_input(prompt, default_value, None,
description, False, False, validator)
self.assertTrue(validator.called)
self.assertEqual(inputed_value1, input)
validator.side_effect = [False, True]
raw_input_mock.side_effect = [inputed_value1, inputed_value2]
- input = ambari_server.get_validated_string_input(prompt, default_value, None,
+ input = ambari__server.get_validated_string_input(prompt, default_value, None,
description, False, False, validator)
self.assertEqual(inputed_value2, input)
- @patch.object(ambari_server, "run_os_command")
+ @patch.object(ambari__server, 'run_os_command')
@patch("__builtin__.open")
@patch("os.path.exists")
def test_is_server_runing(self, os_path_exists_mock, open_mock, \
@@ -1883,15 +2077,15 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
f = open_mock.return_value
f.readline.return_value = "111"
run_os_command_mock.return_value = 0, "", ""
- status, pid = ambari_server.is_server_runing()
+ status, pid = ambari__server.is_server_runing()
self.assertTrue(status)
self.assertEqual(111, pid)
os_path_exists_mock.return_value = False
- status, pid = ambari_server.is_server_runing()
+ status, pid = ambari__server.is_server_runing()
self.assertFalse(status)
- @patch.object(ambari_server, "run_os_command")
+ @patch.object(ambari__server, 'run_os_command')
@patch("__builtin__.open")
@patch("os.path.exists")
def test_is_server_runing_bad_file(self, os_path_exists_mock, open_mock, \
@@ -1900,144 +2094,169 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
f = open_mock.return_value
f.readline.return_value = "" # empty file content
run_os_command_mock.return_value = 0, "", ""
- self.assertRaises(NonFatalException, ambari_server.is_server_runing)
+
+ try:
+ ambari__server.is_server_runing()
+ self.fail("Should throw NonFatalException")
+ except NonFatalException as fe:
+ pass
open_mock.side_effect = IOError('[Errno 13] Permission denied: /var/run/ambari-server/ambari-server.pid')
- self.assertRaises(FatalException, ambari_server.is_server_runing)
+
+ try:
+ ambari__server.is_server_runing()
+ self.fail("Should throw FatalException")
+ except FatalException as fe:
+ pass
+ @patch.object(OSCheck, "get_os_family")
+ @patch.object(OSCheck, "get_os_type")
+ @patch.object(OSCheck, "get_os_major_version")
@patch("os.chdir")
- @patch.object(ambari_server, "run_os_command")
- def test_install_jdk(self, run_os_command_mock, os_chdir_mock):
- run_os_command_
<TRUNCATED>
[7/7] ambari git commit: AMBARI-8373 Refactor the OS-dependent Ambari
Server Windows components
Posted by fb...@apache.org.
AMBARI-8373 Refactor the OS-dependent Ambari Server Windows components
Refactored the Ambari Server setup+reset+update.
+Pervasively adopted the OS-independent factory-based object model
+Unwired the setup-related code from ambari-server.py into separate files.
+Fixed the unit tests
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2914d681
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2914d681
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2914d681
Branch: refs/heads/trunk
Commit: 2914d681dacc675a0a4a379fd20066cfd415e8b8
Parents: bee9379
Author: Florian Barca <fb...@HW10923.local>
Authored: Fri Dec 19 11:41:49 2014 -0800
Committer: Florian Barca <fb...@HW10923.local>
Committed: Fri Dec 19 11:41:49 2014 -0800
----------------------------------------------------------------------
.../src/main/python/ambari_agent/HostInfo.py | 42 +
.../src/main/python/ambari_commons/firewall.py | 93 +-
.../main/python/ambari_commons/inet_utils.py | 17 +-
.../python/ambari_commons/os_family_impl.py | 35 +-
.../src/main/python/ambari_commons/os_linux.py | 13 +-
.../src/main/python/ambari_commons/os_utils.py | 20 +-
.../main/python/ambari_commons/os_windows.py | 147 +-
.../core/providers/windows/system.py | 4 +-
.../functions/get_unique_id_and_date.py | 33 +-
ambari-server/conf/unix/ambari.properties | 17 +-
ambari-server/conf/windows/ambari.properties | 15 +-
.../src/main/python/ambari-server-windows.py | 78 +-
ambari-server/src/main/python/ambari-server.py | 1272 +-------
.../python/ambari_server/dbConfiguration.py | 318 +-
.../ambari_server/dbConfiguration_linux.py | 1189 +++----
.../ambari_server/dbConfiguration_windows.py | 242 +-
.../src/main/python/ambari_server/properties.py | 15 +-
.../python/ambari_server/resourceFilesKeeper.py | 4 +-
.../python/ambari_server/serverConfiguration.py | 277 +-
.../ambari_server/serverConfiguration_linux.py | 67 -
.../serverConfiguration_windows.py | 98 -
.../main/python/ambari_server/serverSetup.py | 1038 ++++--
.../python/ambari_server/serverSetup_linux.py | 795 -----
.../python/ambari_server/serverSetup_windows.py | 313 --
.../main/python/ambari_server/setupSecurity.py | 111 +-
.../src/main/python/ambari_server/userInput.py | 2 +-
.../custom_actions/scripts/check_host.py | 24 +-
.../src/test/python/TestAmbariServer.py | 3066 ++++++++++--------
.../test/python/custom_actions/TestCheckHost.py | 14 +-
29 files changed, 4149 insertions(+), 5210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-agent/src/main/python/ambari_agent/HostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
index a99a85d..5789160 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
@@ -113,6 +113,48 @@ class HostInfo(object):
pass
return False
+ def getTransparentHugePage(self):
+ """
+ Base declaration. The derived classes are supposed to override it and provide OS-specific implementations.
+ :return:
+ """
+ pass
+
+ def checkUsers(self, users, results):
+ """
+ Base declaration. The derived classes are supposed to override it and provide OS-specific implementations.
+ :return:
+ """
+ pass
+
+ def checkIptables(self):
+ """
+ Base declaration. The derived classes are supposed to override it and provide OS-specific implementations.
+ :return:
+ """
+ pass
+
+ def javaProcs(self, list):
+ """
+ Base declaration. The derived classes are supposed to override it and provide OS-specific implementations.
+ :return:
+ """
+ pass
+
+ def getServiceStatus(self, serivce_name):
+ """
+ Base declaration. The derived classes are supposed to override it and provide OS-specific implementations.
+ :return:
+ """
+ pass
+
+ def register(self, dict, componentsMapped=True, commandsInProgress=True):
+ """
+ Base declaration. The derived classes are supposed to override it and provide OS-specific implementations.
+ :return:
+ """
+ pass
+
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
class HostInfoLinux(HostInfo):
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/firewall.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/firewall.py b/ambari-common/src/main/python/ambari_commons/firewall.py
index b73cc0c..448d24e 100644
--- a/ambari-common/src/main/python/ambari_commons/firewall.py
+++ b/ambari-common/src/main/python/ambari_commons/firewall.py
@@ -21,6 +21,8 @@ limitations under the License.
import subprocess
import shlex
from ambari_commons import OSCheck, OSConst
+from ambari_commons.logging_utils import print_warning_msg
+from ambari_commons.os_family_impl import OsFamilyImpl
class Firewall(object):
@@ -31,6 +33,16 @@ class Firewall(object):
self.OS_FAMILY = OSCheck.get_os_family()
def getFirewallObject(self):
+ pass
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class FirewallWindows(Firewall):
+ def getFirewallObject(self):
+ return WindowsFirewallChecks()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class FirewallLinux(Firewall):
+ def getFirewallObject(self):
if self.OS_TYPE == OSConst.OS_UBUNTU:
return UbuntuFirewallChecks()
elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18:
@@ -49,23 +61,31 @@ class FirewallChecks(object):
self.returncode = None
self.stdoutdata = None
self.stderrdata = None
+ # stdout message
+ self.MESSAGE_CHECK_FIREWALL = 'Checking iptables...'
def get_command(self):
return "%s %s %s" % (self.SERVICE_CMD, self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD)
- def check_result(self, retcode, out, err):
+ def check_result(self):
result = False
- if retcode == 3:
+ if self.returncode == 3:
result = False
- elif retcode == 0:
- if "Table: filter" in out:
+ elif self.returncode == 0:
+ if "Table: filter" in self.stdoutdata:
result = True
return result
+ def run_command(self):
+ retcode, out, err = self.run_os_command(self.get_command())
+ self.returncode = retcode
+ self.stdoutdata = out
+ self.stderrdata = err
+
def check_iptables(self):
try:
- retcode, out, err = self.run_os_command(self.get_command())
- return self.check_result(retcode, out, err)
+ self.run_command()
+ return self.check_result()
except OSError:
return False
@@ -76,11 +96,7 @@ class FirewallChecks(object):
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdoutdata, stderrdata) = process.communicate()
- self.returncode = process.returncode
- self.stdoutdata = stdoutdata
- self.stderrdata = stderrdata
- return self.returncode, self.stdoutdata, self.stderrdata
-
+ return process.returncode, stdoutdata, stderrdata
class UbuntuFirewallChecks(FirewallChecks):
@@ -91,13 +107,13 @@ class UbuntuFirewallChecks(FirewallChecks):
def get_command(self):
return "%s %s" % (self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD)
- def check_result(self, retcode, out, err):
+ def check_result(self):
# On ubuntu, the status command returns 0 whether running or not
result = False
- if retcode == 0:
- if "Status: inactive" in out:
+ if self.returncode == 0:
+ if "Status: inactive" in self.stdoutdata:
result = False
- elif "Status: active" in out:
+ elif "Status: active" in self.stdoutdata:
result = True
return result
@@ -108,10 +124,10 @@ class Fedora18FirewallChecks(FirewallChecks):
def get_command(self):
return "systemctl is-active %s" % (self.FIREWALL_SERVICE_NAME)
- def check_result(self, retcode, out, err):
+ def check_result(self):
result = False
- if retcode == 0:
- if "active" in out:
+ if self.returncode == 0:
+ if "active" in self.stdoutdata:
result = True
return result
@@ -123,11 +139,44 @@ class SuseFirewallChecks(FirewallChecks):
def get_command(self):
return "%s %s" % (self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD)
- def check_result(self, retcode, out, err):
+ def check_result(self):
result = False
- if retcode == 0:
- if "SuSEfirewall2 not active" in out:
+ if self.returncode == 0:
+ if "SuSEfirewall2 not active" in self.stdoutdata:
result = False
- elif "### iptables" in out:
+ elif "### iptables" in self.stdoutdata:
result = True
return result
+
+
+class WindowsFirewallChecks(FirewallChecks):
+ def __init__(self):
+ super(WindowsFirewallChecks, self).__init__()
+ self.MESSAGE_CHECK_FIREWALL = 'Checking firewall status...'
+
+ def run_command(self):
+ from ambari_commons.os_windows import run_powershell_script, CHECK_FIREWALL_SCRIPT
+
+ retcode, out, err = run_powershell_script(CHECK_FIREWALL_SCRIPT)
+ self.returncode = retcode
+ self.stdoutdata = out
+ self.stderrdata = err
+
+ def check_result(self):
+ if self.returncode != 0:
+ print_warning_msg("Unable to check firewall status:{0}".format(self.stderrdata))
+ return False
+ profiles_status = [i for i in self.stdoutdata.split("\n") if not i == ""]
+ if "1" in profiles_status:
+ enabled_profiles = []
+ if profiles_status[0] == "1":
+ enabled_profiles.append("DomainProfile")
+ if profiles_status[1] == "1":
+ enabled_profiles.append("StandardProfile")
+ if profiles_status[2] == "1":
+ enabled_profiles.append("PublicProfile")
+ print_warning_msg(
+ "Following firewall profiles are enabled:{0}. Make sure that the firewall is properly configured.".format(
+ ",".join(enabled_profiles)))
+ return False
+ return True
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/inet_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py b/ambari-common/src/main/python/ambari_commons/inet_utils.py
index 2a54cb6..804c63f 100644
--- a/ambari-common/src/main/python/ambari_commons/inet_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py
@@ -22,8 +22,9 @@ import os
import sys
import urllib2
-from exceptions import *
-from logging_utils import *
+from ambari_commons.exceptions import FatalException
+from ambari_commons.logging_utils import print_info_msg, print_warning_msg
+
def download_file(link, destination, chunk_size=16 * 1024):
print_info_msg("Downloading {0} to {1}".format(link, destination))
@@ -34,6 +35,10 @@ def download_file(link, destination, chunk_size=16 * 1024):
force_download_file(link, destination, chunk_size)
+def download_progress_stub(file_name, downloaded_size, blockSize, totalSize):
+ #Facilitates mocking. Normally, the callback used by force_download_file eludes the mocks.
+ download_progress(file_name, downloaded_size, blockSize, totalSize)
+
def download_progress(file_name, downloaded_size, blockSize, totalSize):
percent = int(downloaded_size * 100 / totalSize)
status = "\r" + file_name
@@ -56,7 +61,7 @@ def find_range_components(meta):
if len(range_comp1) > 1:
range_comp2 = range_comp1[0].split(' ') #split away the "bytes" prefix
if len(range_comp2) == 0:
- raise FatalException(12, 'Malformed Content-Range response header: "{}".' % hdr_range)
+ raise FatalException(12, 'Malformed Content-Range response header: "{0}".' % hdr_range)
range_comp3 = range_comp2[1].split('-')
seek_pos = int(range_comp3[0])
if range_comp1[1] != '*': #'*' == unknown length
@@ -72,12 +77,12 @@ def find_range_components(meta):
return (file_size, seek_pos)
-def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func = download_progress):
+def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func = download_progress_stub):
request = urllib2.Request(link)
if os.path.exists(destination) and not os.path.isfile(destination):
#Directory specified as target? Must be a mistake. Bail out, don't assume anything.
- err = 'Download target {} is a directory.' % destination
+ err = 'Download target {0} is a directory.' % destination
raise FatalException(1, err)
(dest_path, file_name) = os.path.split(destination)
@@ -138,7 +143,7 @@ def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func
downloaded_size = os.stat(temp_dest).st_size
if downloaded_size != file_size:
- err = 'Size of downloaded file {} is {} bytes, it is probably damaged or incomplete' % (destination, downloaded_size)
+ err = 'Size of downloaded file {0} is {0} bytes, it is probably damaged or incomplete' % (destination, downloaded_size)
raise FatalException(1, err)
# when download is complete -> mv temp_dest destination
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/os_family_impl.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/os_family_impl.py b/ambari-common/src/main/python/ambari_commons/os_family_impl.py
index f8a3379..d70d942 100644
--- a/ambari-common/src/main/python/ambari_commons/os_family_impl.py
+++ b/ambari-common/src/main/python/ambari_commons/os_family_impl.py
@@ -17,15 +17,15 @@ limitations under the License.
'''
import types
-from os_check import OSCheck
+from ambari_commons import OSCheck
class OsFamilyImpl(object):
"""
- Base class for os depended factory. Usage::
+ Base class for os dependent factory. Usage::
class BaseFoo(object): pass
- @Factory("windows")
+ @OsFamilyImpl("windows")
class OsFoo(object):pass
print BaseFoo()# OsFoo
@@ -62,3 +62,32 @@ class OsFamilyImpl(object):
base_cls.__new__ = types.MethodType(new, base_cls)
return cls
+
+class OsFamilyFuncImpl(object):
+ """
+ Base class for os dependent function. Usage::
+
+ @OSFamilyFuncImpl("windows")
+ def os_foo(...):pass
+
+ """
+ _func_impls = {}
+
+ def _createFunctionInstance(self, func):
+ self._func_impls[func.__module__ + "." + func.__name__ + "." + self.os_const] = func
+
+ def thunk(*args, **kwargs):
+ fn_id_base = func.__module__ + "." + func.__name__
+ fn_id = fn_id_base + "." + OSCheck.get_os_family()
+ if fn_id not in self._func_impls:
+ fn_id = fn_id_base + "." + OsFamilyImpl.DEFAULT
+
+ fn = self._func_impls[fn_id]
+ return fn(*args, **kwargs)
+ return thunk
+
+ def __init__(self, os_family):
+ self.os_const = os_family
+
+ def __call__(self, func):
+ return self._createFunctionInstance(func)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/os_linux.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/os_linux.py b/ambari-common/src/main/python/ambari_commons/os_linux.py
index 38f3fb9..eca3c5d 100644
--- a/ambari-common/src/main/python/ambari_commons/os_linux.py
+++ b/ambari-common/src/main/python/ambari_commons/os_linux.py
@@ -24,7 +24,7 @@ import pwd
import shlex
import subprocess
-from logging_utils import *
+from ambari_commons.logging_utils import print_info_msg, print_warning_msg
NR_CHMOD_CMD = 'chmod {0} {1} {2}'
@@ -33,14 +33,15 @@ NR_CHOWN_CMD = 'chown {0} {1} {2}'
ULIMIT_CMD = "ulimit -n"
-def run_os_command(cmd):
+def os_run_os_command(cmd, env=None):
print_info_msg('about to run command: ' + str(cmd))
if type(cmd) == str:
cmd = shlex.split(cmd)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
- stderr=subprocess.PIPE
+ stderr=subprocess.PIPE,
+ env=env
)
(stdoutdata, stderrdata) = process.communicate()
return process.returncode, stdoutdata, stderrdata
@@ -64,17 +65,17 @@ def os_set_file_permissions(file, mod, recursive, user):
else:
params = ""
command = NR_CHMOD_CMD.format(params, mod, file)
- retcode, out, err = run_os_command(command)
+ retcode, out, err = os_run_os_command(command)
if retcode != 0:
print_warning_msg(WARN_MSG.format(command, file, err))
command = NR_CHOWN_CMD.format(params, user, file)
- retcode, out, err = run_os_command(command)
+ retcode, out, err = os_run_os_command(command)
if retcode != 0:
print_warning_msg(WARN_MSG.format(command, file, err))
def os_set_open_files_limit(maxOpenFiles):
command = "%s %s" % (ULIMIT_CMD, str(maxOpenFiles))
- run_os_command(command)
+ os_run_os_command(command)
def os_getpass(prompt):
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/os_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/os_utils.py b/ambari-common/src/main/python/ambari_commons/os_utils.py
index 3f4819d..6e2c1d2 100644
--- a/ambari-common/src/main/python/ambari_commons/os_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/os_utils.py
@@ -18,20 +18,23 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
+import os
import shutil
import string
-import os
-from os_check import *
+from ambari_commons import OSCheck
if OSCheck.is_windows_family():
- from os_windows import *
+ from ambari_commons.os_windows import os_change_owner, os_getpass, os_is_root, os_run_os_command, \
+ os_set_open_files_limit, os_set_file_permissions
else:
# MacOS not supported
- from os_linux import *
+ from ambari_commons.os_linux import os_change_owner, os_getpass, os_is_root, os_run_os_command, \
+ os_set_open_files_limit, os_set_file_permissions
+ pass
-from logging_utils import *
-from exceptions import FatalException
+from ambari_commons.exceptions import FatalException
+from ambari_commons.logging_utils import print_info_msg, print_warning_msg
def is_valid_filepath(filepath):
@@ -88,6 +91,9 @@ def set_file_permissions(file, mod, user, recursive):
else:
print_info_msg("File %s does not exist" % file)
+def run_os_command(cmd, env=None):
+ return os_run_os_command(cmd, env)
+
def is_root():
return os_is_root()
@@ -108,4 +114,4 @@ def find_in_path(file):
full_path = os.path.join(dir, file)
if os.path.exists(full_path):
return full_path
- raise Exception("File {} not found in PATH".format(file))
\ No newline at end of file
+ raise Exception("File {} not found in PATH".format(file))
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/ambari_commons/os_windows.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/os_windows.py b/ambari-common/src/main/python/ambari_commons/os_windows.py
index 2fb98e4..0cc4ea2 100644
--- a/ambari-common/src/main/python/ambari_commons/os_windows.py
+++ b/ambari-common/src/main/python/ambari_commons/os_windows.py
@@ -17,41 +17,35 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
-import os
import getpass
+import os
+import random
import shlex
import subprocess
import sys
+import tempfile
import time
-import win32api
-import win32event
-import win32service
-import win32con
-import win32serviceutil
-import wmi
-import random
import string
import ctypes
-from win32security import *
-from win32api import *
-from winerror import ERROR_INVALID_HANDLE
-from win32process import GetExitCodeProcess, STARTF_USESTDHANDLES, STARTUPINFO, CreateProcessAsUser
-from win32event import WaitForSingleObject, INFINITE
import msvcrt
-import tempfile
-from win32event import *
-from win32api import CloseHandle
-
-from ambari_commons.exceptions import *
-from logging_utils import *
-from win32security import LsaOpenPolicy, POLICY_CREATE_ACCOUNT, POLICY_LOOKUP_NAMES, LookupAccountName, \
- LsaAddAccountRights, LsaRemoveAccountRights, SE_SERVICE_LOGON_NAME
-from win32net import NetUserAdd
-from win32netcon import USER_PRIV_USER, UF_NORMAL_ACCOUNT, UF_SCRIPT
import pywintypes
+import winerror
+import win32api
+import win32con
+import win32event
+import win32net
+import win32netcon
+import win32process
+import win32security
+import win32service
+import win32serviceutil
+import wmi
+
+from ambari_commons.exceptions import FatalException
+from ambari_commons.logging_utils import print_info_msg, print_warning_msg
SERVICE_STATUS_UNKNOWN = "unknown"
SERVICE_STATUS_STARTING = "starting"
@@ -77,17 +71,17 @@ class OSVERSIONINFOEXW(ctypes.Structure):
('wReserved', ctypes.c_byte)]
def get_windows_version():
- """
- Get's the OS major and minor versions. Returns a tuple of
- (OS_MAJOR, OS_MINOR).
- """
- os_version = OSVERSIONINFOEXW()
- os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version)
- retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version))
- if retcode != 0:
- raise Exception("Failed to get OS version")
-
- return os_version.dwMajorVersion, os_version.dwMinorVersion, os_version.dwBuildNumber
+ """
+ Gets the OS major and minor versions. Returns a tuple of
+ (OS_MAJOR, OS_MINOR).
+ """
+ os_version = OSVERSIONINFOEXW()
+ os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version)
+ retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version))
+ if retcode != 0:
+ raise Exception("Failed to get OS version")
+
+ return os_version.dwMajorVersion, os_version.dwMinorVersion, os_version.dwBuildNumber
CHECK_FIREWALL_SCRIPT = """[string]$CName = $env:computername
$reg = [Microsoft.Win32.RegistryKey]::OpenRemoteBaseKey("LocalMachine",$computer)
@@ -116,25 +110,25 @@ def _get_files_output(out, err):
def _safe_duplicate_handle(h):
try:
- h = DuplicateHandle(GetCurrentProcess(),
- h,
- GetCurrentProcess(),
- 0,
- True,
- win32con.DUPLICATE_SAME_ACCESS)
+ h = win32api.DuplicateHandle(win32process.GetCurrentProcess(),
+ h,
+ win32process.GetCurrentProcess(),
+ 0,
+ True,
+ win32con.DUPLICATE_SAME_ACCESS)
return True, h
except Exception as exc:
- if exc.winerror == ERROR_INVALID_HANDLE:
+ if exc.winerror == winerror.ERROR_INVALID_HANDLE:
return True, None
return False, None
def run_os_command_impersonated(cmd, user, password, domain='.'):
- si = STARTUPINFO()
+ si = win32process.STARTUPINFO()
out_handle, err_handle, out_file, err_file = _create_tmp_files()
- ok, si.hStdInput = _safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE))
+ ok, si.hStdInput = _safe_duplicate_handle(win32api.GetStdHandle(win32api.STD_INPUT_HANDLE))
if not ok:
raise Exception("Unable to create StdInput for child process")
@@ -145,27 +139,29 @@ def run_os_command_impersonated(cmd, user, password, domain='.'):
if not ok:
raise Exception("Unable to create StdErr for child process")
- si.dwFlags = STARTF_USESTDHANDLES
+ si.dwFlags = win32process.STARTF_USESTDHANDLES
si.lpDesktop = ""
- user_token = LogonUser(user, domain, password, win32con.LOGON32_LOGON_SERVICE, win32con.LOGON32_PROVIDER_DEFAULT)
- primary_token = DuplicateTokenEx(user_token, SecurityImpersonation, 0, TokenPrimary)
- info = CreateProcessAsUser(primary_token, None, cmd, None, None, 1, 0, None, None, si)
+ user_token = win32security.LogonUser(user, domain, password,
+ win32con.LOGON32_LOGON_SERVICE, win32con.LOGON32_PROVIDER_DEFAULT)
+ primary_token = win32security.DuplicateTokenEx(user_token,
+ win32security.SecurityImpersonation, 0, win32security.TokenPrimary)
+ info = win32process.CreateProcessAsUser(primary_token, None, cmd, None, None, 1, 0, None, None, si)
hProcess, hThread, dwProcessId, dwThreadId = info
hThread.Close()
try:
- WaitForSingleObject(hProcess, INFINITE)
+ win32event.WaitForSingleObject(hProcess, win32event.INFINITE)
except KeyboardInterrupt:
pass
out, err = _get_files_output(out_file, err_file)
- exitcode = GetExitCodeProcess(hProcess)
+ exitcode = win32process.GetExitCodeProcess(hProcess)
return exitcode, out, err
-def run_os_command(cmd, env=None):
+def os_run_os_command(cmd, env=None):
if isinstance(cmd,basestring):
cmd = cmd.replace("\\", "\\\\")
cmd = shlex.split(cmd)
@@ -186,13 +182,13 @@ def run_powershell_script(script_content):
script_file = open(os.path.join(tmp_dir,random_filename+".ps1"),"w")
script_file.write(script_content)
script_file.close()
- result = run_os_command("powershell -ExecutionPolicy unrestricted -File {0}".format(script_file.name))
+ result = os_run_os_command("powershell -ExecutionPolicy unrestricted -File {0}".format(script_file.name))
os.remove(script_file.name)
return result
def os_change_owner(filePath, user):
cmd = ['icacls', filePath, '/setowner', user]
- retcode, outdata, errdata = run_os_command(cmd)
+ retcode, outdata, errdata = os_run_os_command(cmd)
return retcode
def os_is_root():
@@ -200,9 +196,10 @@ def os_is_root():
Checks whether the current user is a member of the Administrators group
Returns True if yes, otherwise False
'''
- retcode, out, err = run_os_command(WHOAMI_GROUPS)
+ retcode, out, err = os_run_os_command(WHOAMI_GROUPS)
if retcode != 0:
- err_msg = "Unable to check the current user's group memberships. Command {0} returned exit code {1} with message: {2}".format(WHOAMI_GROUPS, retcode, err)
+ err_msg = "Unable to check the current user's group memberships. " \
+ "Command {0} returned exit code {1} with message: {2}".format(WHOAMI_GROUPS, retcode, err)
print_warning_msg(err_msg)
raise FatalException(retcode, err_msg)
@@ -221,20 +218,20 @@ def os_set_file_permissions(file, mod, recursive, user):
#else:
# params = ""
#command = NR_CHMOD_CMD.format(params, mod, file)
- #retcode, out, err = run_os_command(command)
+ #retcode, out, err = os_run_os_command(command)
#if retcode != 0:
# print_warning_msg(WARN_MSG.format(command, file, err))
#command = NR_CHOWN_CMD.format(params, user, file)
- #retcode, out, err = run_os_command(command)
+ #retcode, out, err = os_run_os_command(command)
#if retcode != 0:
# print_warning_msg(WARN_MSG.format(command, file, err))
# rights = mod
# acls_remove_cmd = "icacls {0} /remove {1}".format(file, user)
- # retcode, out, err = run_os_command(acls_remove_cmd)
+ # retcode, out, err = os_run_os_command(acls_remove_cmd)
# if retcode == 0:
# acls_modify_cmd = "icacls {0} /grant {1}:{2}".format(file, user, rights)
- # retcode, out, err = run_os_command(acls_modify_cmd)
+ # retcode, out, err = os_run_os_command(acls_modify_cmd)
return retcode
@@ -248,8 +245,6 @@ def os_getpass(prompt, stream=None):
if sys.stdin is not sys.__stdin__:
return getpass.fallback_getpass(prompt, stream)
- import msvcrt
-
for c in prompt:
msvcrt.putch(c)
@@ -487,7 +482,8 @@ class WinService(win32serviceutil.ServiceFramework):
def _StopOrWaitForChildProcessToFinish(self, childProcess):
#Wait for the child process to finish or for the stop event to be signaled
- if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([self._heventSvcStop, childProcess._handle], False, win32event.INFINITE)):
+ if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([self._heventSvcStop, childProcess._handle],
+ False, win32event.INFINITE)):
# The OS only detaches the child process when the master process exits.
# We must kill it manually.
try:
@@ -503,24 +499,24 @@ class WinService(win32serviceutil.ServiceFramework):
class SystemWideLock(object):
def __init__(self, name):
- self._mutex = CreateMutex(None, 0, name)
+ self._mutex = win32event.CreateMutex(None, 0, name)
def lock(self, timeout=0):
- result = WaitForSingleObject(self._mutex, timeout)
- if result in [WAIT_TIMEOUT, WAIT_ABANDONED, WAIT_FAILED]:
+ result = win32event.WaitForSingleObject(self._mutex, timeout)
+ if result in [win32event.WAIT_TIMEOUT, win32event.WAIT_ABANDONED, win32event.WAIT_FAILED]:
return False
- elif result == WAIT_OBJECT_0:
+ elif result == win32event.WAIT_OBJECT_0:
return True
def unlock(self):
try:
- ReleaseMutex(self._mutex)
+ win32event.ReleaseMutex(self._mutex)
return True
except:
return False
def __del__(self):
- CloseHandle(self._mutex)
+ win32api.CloseHandle(self._mutex)
class UserHelper(object):
ACTION_OK = 0
@@ -528,17 +524,18 @@ class UserHelper(object):
ACTION_FAILED = -1
def __init__(self):
- self._policy = LsaOpenPolicy(None, POLICY_CREATE_ACCOUNT | POLICY_LOOKUP_NAMES)
+ self._policy = win32security.LsaOpenPolicy(None,
+ win32security.POLICY_CREATE_ACCOUNT | win32security.POLICY_LOOKUP_NAMES)
def create_user(self, name, password, comment="Ambari user"):
user_info = {}
user_info['name'] = name
user_info['password'] = password
- user_info['priv'] = USER_PRIV_USER
+ user_info['priv'] = win32netcon.USER_PRIV_USER
user_info['comment'] = comment
- user_info['flags'] = UF_NORMAL_ACCOUNT | UF_SCRIPT
+ user_info['flags'] = win32netcon.UF_NORMAL_ACCOUNT | win32netcon.UF_SCRIPT
try:
- NetUserAdd(None, 1, user_info)
+ win32net.NetUserAdd(None, 1, user_info)
except pywintypes.error as e:
if e.winerror == 2224:
return UserHelper.USER_EXISTS, e.strerror
@@ -548,16 +545,16 @@ class UserHelper(object):
def add_user_privilege(self, name, privilege):
try:
- acc_sid = LookupAccountName(None, name)[0]
- LsaAddAccountRights(self._policy, acc_sid, (privilege,))
+ acc_sid = win32security.LookupAccountName(None, name)[0]
+ win32security.LsaAddAccountRights(self._policy, acc_sid, (privilege,))
except pywintypes.error as e:
return UserHelper.ACTION_FAILED, e.strerror
return UserHelper.ACTION_OK, "Privilege added."
def remove_user_privilege(self, name, privilege):
try:
- acc_sid = LookupAccountName(None, name)[0]
- LsaRemoveAccountRights(self._policy, acc_sid, 0, (privilege,))
+ acc_sid = win32security.LookupAccountName(None, name)[0]
+ win32security.LsaRemoveAccountRights(self._policy, acc_sid, 0, (privilege,))
except pywintypes.error as e:
return UserHelper.ACTION_FAILED, e.strerror
return UserHelper.ACTION_OK, "Privilege removed."
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/resource_management/core/providers/windows/system.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/providers/windows/system.py b/ambari-common/src/main/python/resource_management/core/providers/windows/system.py
index e7a98fc..dac44ec 100644
--- a/ambari-common/src/main/python/resource_management/core/providers/windows/system.py
+++ b/ambari-common/src/main/python/resource_management/core/providers/windows/system.py
@@ -140,7 +140,7 @@ def QueryPrivilegeState(hToken, priv):
for (id, attr) in privList:
if id == privId:
privState = attr
- Logger.debug('Privilege state: {}={} ({}) Enabled={}'.format(privId, priv, LookupPrivilegeDisplayName(None, priv), privState))
+ Logger.debug('Privilege state: {0}={1} ({2}) Enabled={3}'.format(privId, priv, LookupPrivilegeDisplayName(None, priv), privState))
return privState
# Execute command. As windows hdp stack heavily relies on proper environment it is better to reload fresh environment
@@ -182,7 +182,7 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=
if not ok:
raise Exception("Unable to create StdErr for child process")
- Logger.debug("Redirecting stdout to '{}', stderr to '{}'".format(out_file.name, err_file.name))
+ Logger.debug("Redirecting stdout to '{0}', stderr to '{1}'".format(out_file.name, err_file.name))
si.dwFlags = win32con.STARTF_USESTDHANDLES
si.lpDesktop = ""
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py
index afc82bb..3f8e983 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py
@@ -23,17 +23,30 @@ Ambari Agent
__all__ = ["get_unique_id_and_date"]
import datetime
from resource_management.core import shell
-from ambari_commons import os_check
+from ambari_commons import os_check, OSConst
+from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl
+
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def get_volume_serial_number():
+ from ambari_commons.os_windows import os_run_os_command
+
+ id = ""
+ code, out, err = os_run_os_command("cmd /c vol C:")
+ for line in out.splitlines():
+ if line.startswith(" Volume Serial Number is"):
+ id = line[25:]
+
+ return id
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def get_volume_serial_number():
+ out = shell.checked_call("hostid")[1].split('\n')[-1] # bugfix: take the lastline (stdin is not tty part cut)
+ id = out.strip()
+ return id
+
def get_unique_id_and_date():
- if os_check.OSCheck.is_windows_family():
- from ambari_commons.os_windows import run_os_command
- code, out, err = run_os_command("cmd /c vol C:")
- for line in out.splitlines():
- if line.startswith(" Volume Serial Number is"):
- id = line[25:]
- else:
- out = shell.checked_call("hostid")[1].split('\n')[-1] # bugfix: take the lastline (stdin is not tty part cut)
- id = out.strip()
+ id = get_volume_serial_number()
now = datetime.datetime.now()
date = now.strftime("%M%d%y")
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/conf/unix/ambari.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties
index e29a6e2..84c96a3 100644
--- a/ambari-server/conf/unix/ambari.properties
+++ b/ambari-server/conf/unix/ambari.properties
@@ -22,10 +22,23 @@ security.server.keys_dir = /var/lib/ambari-server/keys
resources.dir = /var/lib/ambari-server/resources
shared.resources.dir = /usr/lib/ambari-server/lib/ambari_commons/resources
custom.action.definitions = /var/lib/ambari-server/resources/custom_action_definitions
+
+java.releases=jdk1.6,jdk1.7
+jdk1.6.desc=Oracle JDK 1.6
jdk1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-6u31-linux-x64.bin
-jce_policy1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip
+jdk1.6.dest-file=jdk-6u31-linux-x64.bin
+jdk1.6.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip
+jdk1.6.jcpol-file=jce_policy-6.zip
+jdk1.6.home=/usr/jdk64/
+jdk1.6.re=Creating (jdk.*)/jre
+jdk1.7.desc=Oracle JDK 1.7
jdk1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-linux-x64.tar.gz
-jce_policy1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip
+jdk1.7.dest-file=jdk-7u67-linux-x64.tar.gz
+jdk1.7.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip
+jdk1.7.jcpol-file=UnlimitedJCEPolicyJDK7.zip
+jdk1.7.home=/usr/jdk64/
+jdk1.7.re=(jdk.*)/jre
+
metadata.path=/var/lib/ambari-server/resources/stacks
common.services.path=/var/lib/ambari-server/resources/common-services
server.version.file=/var/lib/ambari-server/resources/version
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/conf/windows/ambari.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties
index 3982bb9..434e42a 100644
--- a/ambari-server/conf/windows/ambari.properties
+++ b/ambari-server/conf/windows/ambari.properties
@@ -23,13 +23,14 @@ custom.action.definitions=resources\\custom_action_definitions
#Comma-separated list of JDK versions
#java.releases=jdk1.8.20,jdk1.6.31
-java.releases=jdk1.7.67
-jdk1.7.67.desc=Oracle JDK 1.7.67
-jdk1.7.67.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe
-jdk1.7.67.dest-file=jdk-7u67-windows-x64.exe
-jdk1.7.67.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip
-jdk1.7.67.jcpol-file=UnlimitedJCEPolicyJDK7.zip
-jdk1.7.67.home=C:\\jdk1.7.0_67
+java.releases=jdk1.7
+jdk1.7.desc=Oracle JDK 1.7
+jdk1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe
+jdk1.7.dest-file=jdk-7u67-windows-x64.exe
+jdk1.7.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip
+jdk1.7.jcpol-file=UnlimitedJCEPolicyJDK7.zip
+jdk1.7.home=C:\\jdk1.7.0_67
+jdk1.7.re=(jdk.*)/jre
metadata.path=resources\\stacks
common.services.path=resources\\common-services
http://git-wip-us.apache.org/repos/asf/ambari/blob/2914d681/ambari-server/src/main/python/ambari-server-windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server-windows.py b/ambari-server/src/main/python/ambari-server-windows.py
index 6c4f894..8940972 100644
--- a/ambari-server/src/main/python/ambari-server-windows.py
+++ b/ambari-server/src/main/python/ambari-server-windows.py
@@ -19,20 +19,27 @@ limitations under the License.
'''
import optparse
-
-from ambari_commons.ambari_service import AmbariService, ENV_PYTHON_PATH
-from ambari_commons.logging_utils import *
-from ambari_commons.os_utils import remove_file
+import os
+import subprocess
+import sys
+
+from ambari_commons.ambari_service import AmbariService
+from ambari_commons.exceptions import NonFatalException, FatalException
+from ambari_commons.logging_utils import print_error_msg, print_info_msg, print_warning_msg
+from ambari_commons.os_utils import remove_file, set_open_files_limit
from ambari_commons.os_windows import SvcStatusCallback
-from ambari_server import utils
-from ambari_server.dbConfiguration import DBMSConfig
+from ambari_server.dbConfiguration import DBMSConfigFactory
from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException
-from ambari_server.serverConfiguration import *
-from ambari_server.serverSetup import setup, reset, is_server_running, upgrade
-from ambari_server.setupActions import *
-from ambari_server.setupSecurity import *
-from ambari_server.serverSetup_windows import SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY
+from ambari_server.serverConfiguration import find_jdk, get_ambari_properties, get_value_from_properties, \
+ get_full_ambari_classpath, configDefaults, VERBOSE_OUTPUT_KEY, DEBUG_MODE_KEY, SUSPEND_START_MODE_KEY, \
+ SERVER_OUT_FILE_KEY, RESOURCES_DIR_PROPERTY, RESOURCES_DIR_DEFAULT, STACK_LOCATION_KEY, STACK_LOCATION_DEFAULT
+from ambari_server.serverSetup import setup, reset, is_server_running, upgrade, SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY
+from ambari_server.setupActions import SETUP_ACTION, START_ACTION, PSTART_ACTION, STOP_ACTION, RESET_ACTION, \
+ STATUS_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, LDAP_SETUP_ACTION, SETUP_SECURITY_ACTION, ACTION_REQUIRE_RESTART
+from ambari_server.setupSecurity import setup_ambari_krb5_jaas, setup_https, setup_ldap, setup_master_key
+from ambari_server.userInput import get_validated_string_input
+from ambari_server.utils import check_reverse_lookup, save_pid
# debug settings
SERVER_START_DEBUG = False
@@ -42,11 +49,6 @@ SUSPEND_START_MODE = False
ambari_provider_module_option = ""
ambari_provider_module = os.environ.get('AMBARI_PROVIDER_MODULE')
-#Common setup or upgrade message
-SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server setup\" command to create the user\n" \
-"- If this is an upgrade of an existing setup, run the \"ambari-server upgrade\" command.\n" \
-"Refer to the Ambari documentation for more information on setup and upgrade."
-
AMBARI_SERVER_DIE_MSG = "Ambari Server java process died with exitcode {0}. Check {1} for more information."
if ambari_provider_module is not None:
@@ -110,7 +112,7 @@ class AmbariServerService(AmbariService):
if not self._StopOrWaitForChildProcessToFinish(childProc):
return
- pid_file_path = PID_DIR + os.sep + PID_NAME
+ pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
remove_file(pid_file_path)
pass
@@ -122,7 +124,7 @@ class AmbariServerService(AmbariService):
outFilePath = properties[SERVER_OUT_FILE_KEY]
if (outFilePath is None or outFilePath == ""):
- outFilePath = SERVER_OUT_FILE
+ outFilePath = configDefaults.SERVER_OUT_FILE
self._RedirectOutputStreamsToFile(outFilePath)
pass
@@ -157,7 +159,7 @@ def start(options):
childProc.wait()
- pid_file_path = PID_DIR + os.sep + PID_NAME
+ pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
remove_file(pid_file_path)
#
@@ -199,7 +201,7 @@ def server_process_main(options, scmStatus=None):
except AttributeError:
pass
- if not utils.check_reverse_lookup():
+ if not check_reverse_lookup():
print_warning_msg("The hostname was not found in the reverse DNS lookup. "
"This may result in incorrect behavior. "
"Please check the DNS setup and fix the issue.")
@@ -208,12 +210,11 @@ def server_process_main(options, scmStatus=None):
print_info_msg("Ambari Server is not running...")
- conf_dir = get_conf_dir()
jdk_path = find_jdk()
if jdk_path is None:
err = "No JDK found, please run the \"ambari-server setup\" " \
"command to install a JDK automatically or install any " \
- "JDK manually to " + JDK_INSTALL_DIR
+ "JDK manually to " + configDefaults.JDK_INSTALL_DIR
raise FatalException(1, err)
# Preparations
@@ -236,17 +237,15 @@ def server_process_main(options, scmStatus=None):
if scmStatus is not None:
scmStatus.reportStartPending()
- conf_dir = os.path.abspath(conf_dir) + os.pathsep + get_ambari_classpath()
- if conf_dir.find(' ') != -1:
- conf_dir = '"' + conf_dir + '"'
+ conf_dir = get_full_ambari_classpath()
- java_exe = jdk_path + os.sep + JAVA_EXE_SUBPATH
- pidfile = PID_DIR + os.sep + PID_NAME
+ java_exe = os.path.join(jdk_path, configDefaults.JAVA_EXE_SUBPATH)
+ pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
command_base = SERVER_START_CMD_DEBUG if (DEBUG_MODE or SERVER_START_DEBUG) else SERVER_START_CMD
suspend_mode = 'y' if SUSPEND_START_MODE else 'n'
command = command_base.format(conf_dir, suspend_mode)
- if not os.path.exists(PID_DIR):
- os.makedirs(PID_DIR, 0755)
+ if not os.path.exists(configDefaults.PID_DIR):
+ os.makedirs(configDefaults.PID_DIR, 0755)
set_open_files_limit(get_ulimit_open_files());
@@ -264,18 +263,18 @@ def server_process_main(options, scmStatus=None):
if pidJava <= 0:
procJava.terminate()
exitcode = procJava.returncode
- exitfile = os.path.join(PID_DIR, EXITCODE_NAME)
- utils.save_pid(exitcode, exitfile)
+ exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
+ save_pid(exitcode, exitfile)
if scmStatus is not None:
scmStatus.reportStopPending()
- raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, SERVER_OUT_FILE))
+ raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
else:
- utils.save_pid(pidJava, pidfile)
- print "Server PID at: "+pidfile
- print "Server out at: "+SERVER_OUT_FILE
- print "Server log at: "+SERVER_LOG_FILE
+ save_pid(pidJava, pidfile)
+ print "Server PID at: "+ pidfile
+ print "Server out at: "+ configDefaults.SERVER_OUT_FILE
+ print "Server log at: "+ configDefaults.SERVER_LOG_FILE
if scmStatus is not None:
scmStatus.reportStarted()
@@ -289,13 +288,15 @@ def server_process_main(options, scmStatus=None):
#Wait until the status is 'started' or a configured timeout elapses
#If the timeout has been reached, bail out with exception
def ensure_dbms_is_running(options, properties, scmStatus):
- dbms = DBMSConfig.create(options, properties, "Ambari")
+ factory = DBMSConfigFactory()
+
+ dbms = factory.create(options, properties, "Ambari")
if not dbms._is_jdbc_driver_installed(properties):
raise FatalException(-1, "JDBC driver is not installed. Run ambari-server setup and try again.")
dbms.ensure_dbms_is_running(options, properties, scmStatus)
- dbms2 = DBMSConfig.create(options, properties, "Metrics")
+ dbms2 = factory.create(options, properties, "Metrics")
if dbms2.database_host.lower() != dbms.database_host.lower():
dbms2.ensure_dbms_is_running(options, properties, scmStatus)
pass
@@ -499,6 +500,7 @@ def main():
#perform checks
options.warnings = []
+ options.exit_message = None
options.must_set_database_options = False
if are_cmd_line_db_args_blank(options):