You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by fb...@apache.org on 2015/01/24 02:37:12 UTC
[8/8] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari
Server Windows components - Part 1.4
AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4
+Further decomposing ambari-server.py: Moved setup, reset and upgrade into their own files
+Made them OS-independent - now the same routines are being used in Windows too (save upgrade, which is not active in Windows yet)
+Merged the OS-dependent code for JDBC & JDK setup, database interface creation, firewall checks
+Refactored the database setup: split the supported DBMSes into distinct objects
+Replaced recursive calls in the user input routines
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/49955a35
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/49955a35
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/49955a35
Branch: refs/heads/trunk
Commit: 49955a35971f723dcc8d4acee57fbd9d6bec58f5
Parents: ccc58c1
Author: Florian Barca <fb...@hortonworks.com>
Authored: Fri Jan 23 17:36:55 2015 -0800
Committer: Florian Barca <fb...@hortonworks.com>
Committed: Fri Jan 23 17:36:55 2015 -0800
----------------------------------------------------------------------
.../src/main/python/ambari_commons/firewall.py | 90 +-
.../main/python/ambari_commons/inet_utils.py | 6 +-
.../main/python/ambari_commons/logging_utils.py | 16 +-
ambari-server/conf/unix/ambari.properties | 17 +-
ambari-server/conf/windows/ambari.properties | 2 +-
.../src/main/python/ambari-server-windows.py | 51 +-
ambari-server/src/main/python/ambari-server.py | 2789 +---------------
.../python/ambari_server/dbConfiguration.py | 432 ++-
.../ambari_server/dbConfiguration_linux.py | 1300 ++++----
.../ambari_server/dbConfiguration_windows.py | 298 +-
.../python/ambari_server/serverConfiguration.py | 449 ++-
.../main/python/ambari_server/serverSetup.py | 1075 ++++--
.../python/ambari_server/serverSetup_linux.py | 793 -----
.../python/ambari_server/serverSetup_windows.py | 300 --
.../main/python/ambari_server/serverUpgrade.py | 305 ++
.../main/python/ambari_server/setupActions.py | 4 +-
.../main/python/ambari_server/setupSecurity.py | 275 +-
.../src/main/python/ambari_server/userInput.py | 29 +-
.../custom_actions/scripts/check_host.py | 2 +-
.../2.1/hooks/before-ANY/scripts/setup_jdk.py | 4 +-
.../package/scripts/mapred_service_check.py | 4 +-
.../src/test/python/TestAmbariServer.py | 3081 ++++++++++--------
ambari-server/src/test/python/TestOSCheck.py | 7 +-
23 files changed, 4782 insertions(+), 6547 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-common/src/main/python/ambari_commons/firewall.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/firewall.py b/ambari-common/src/main/python/ambari_commons/firewall.py
index 76d9890..311e6d5 100644
--- a/ambari-common/src/main/python/ambari_commons/firewall.py
+++ b/ambari-common/src/main/python/ambari_commons/firewall.py
@@ -18,9 +18,14 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
+import subprocess
+import shlex
from ambari_commons import OSCheck, OSConst
+from ambari_commons.logging_utils import print_warning_msg
+from ambari_commons.os_family_impl import OsFamilyImpl
from ambari_commons.os_utils import run_os_command
+
class Firewall(object):
def __init__(self):
# OS info
@@ -29,6 +34,16 @@ class Firewall(object):
self.OS_FAMILY = OSCheck.get_os_family()
def getFirewallObject(self):
+ pass
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class FirewallWindows(Firewall):
+ def getFirewallObject(self):
+ return WindowsFirewallChecks()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class FirewallLinux(Firewall):
+ def getFirewallObject(self):
if self.OS_TYPE == OSConst.OS_UBUNTU:
return UbuntuFirewallChecks()
elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18:
@@ -47,23 +62,31 @@ class FirewallChecks(object):
self.returncode = None
self.stdoutdata = None
self.stderrdata = None
+ # stdout message
+ self.MESSAGE_CHECK_FIREWALL = 'Checking iptables...'
def get_command(self):
return "%s %s %s" % (self.SERVICE_CMD, self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD)
- def check_result(self, retcode, out, err):
+ def check_result(self):
result = False
- if retcode == 3:
+ if self.returncode == 3:
result = False
- elif retcode == 0:
- if "Table: filter" in out:
+ elif self.returncode == 0:
+ if "Table: filter" in self.stdoutdata:
result = True
return result
+ def run_command(self):
+ retcode, out, err = run_os_command(self.get_command())
+ self.returncode = retcode
+ self.stdoutdata = out
+ self.stderrdata = err
+
def check_iptables(self):
try:
- self.returncode, self.stdoutdata, self.stderrdata = run_os_command(self.get_command())
- return self.check_result(self.returncode, self.stdoutdata, self.stderrdata)
+ self.run_command()
+ return self.check_result()
except OSError:
return False
@@ -76,13 +99,13 @@ class UbuntuFirewallChecks(FirewallChecks):
def get_command(self):
return "%s %s" % (self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD)
- def check_result(self, retcode, out, err):
+ def check_result(self):
# On ubuntu, the status command returns 0 whether running or not
result = False
- if retcode == 0:
- if "Status: inactive" in out:
+ if self.returncode == 0:
+ if "Status: inactive" in self.stdoutdata:
result = False
- elif "Status: active" in out:
+ elif "Status: active" in self.stdoutdata:
result = True
return result
@@ -93,10 +116,10 @@ class Fedora18FirewallChecks(FirewallChecks):
def get_command(self):
return "systemctl is-active %s" % (self.FIREWALL_SERVICE_NAME)
- def check_result(self, retcode, out, err):
+ def check_result(self):
result = False
- if retcode == 0:
- if "active" in out:
+ if self.returncode == 0:
+ if "active" in self.stdoutdata:
result = True
return result
@@ -108,11 +131,44 @@ class SuseFirewallChecks(FirewallChecks):
def get_command(self):
return "%s %s" % (self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD)
- def check_result(self, retcode, out, err):
+ def check_result(self):
result = False
- if retcode == 0:
- if "SuSEfirewall2 not active" in out:
+ if self.returncode == 0:
+ if "SuSEfirewall2 not active" in self.stdoutdata:
result = False
- elif "### iptables" in out:
+ elif "### iptables" in self.stdoutdata:
result = True
return result
+
+
+class WindowsFirewallChecks(FirewallChecks):
+ def __init__(self):
+ super(WindowsFirewallChecks, self).__init__()
+ self.MESSAGE_CHECK_FIREWALL = 'Checking firewall status...'
+
+ def run_command(self):
+ from ambari_commons.os_windows import run_powershell_script, CHECK_FIREWALL_SCRIPT
+
+ retcode, out, err = run_powershell_script(CHECK_FIREWALL_SCRIPT)
+ self.returncode = retcode
+ self.stdoutdata = out
+ self.stderrdata = err
+
+ def check_result(self):
+ if self.returncode != 0:
+ print_warning_msg("Unable to check firewall status:{0}".format(self.stderrdata))
+ return False
+ profiles_status = [i for i in self.stdoutdata.split("\n") if not i == ""]
+ if "1" in profiles_status:
+ enabled_profiles = []
+ if profiles_status[0] == "1":
+ enabled_profiles.append("DomainProfile")
+ if profiles_status[1] == "1":
+ enabled_profiles.append("StandardProfile")
+ if profiles_status[2] == "1":
+ enabled_profiles.append("PublicProfile")
+ print_warning_msg(
+ "Following firewall profiles are enabled:{0}. Make sure that the firewall is properly configured.".format(
+ ",".join(enabled_profiles)))
+ return False
+ return True
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-common/src/main/python/ambari_commons/inet_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py b/ambari-common/src/main/python/ambari_commons/inet_utils.py
index 2a54cb6..5c9275c 100644
--- a/ambari-common/src/main/python/ambari_commons/inet_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py
@@ -56,7 +56,7 @@ def find_range_components(meta):
if len(range_comp1) > 1:
range_comp2 = range_comp1[0].split(' ') #split away the "bytes" prefix
if len(range_comp2) == 0:
- raise FatalException(12, 'Malformed Content-Range response header: "{}".' % hdr_range)
+ raise FatalException(12, 'Malformed Content-Range response header: "{0}".' % hdr_range)
range_comp3 = range_comp2[1].split('-')
seek_pos = int(range_comp3[0])
if range_comp1[1] != '*': #'*' == unknown length
@@ -77,7 +77,7 @@ def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func
if os.path.exists(destination) and not os.path.isfile(destination):
#Directory specified as target? Must be a mistake. Bail out, don't assume anything.
- err = 'Download target {} is a directory.' % destination
+ err = 'Download target {0} is a directory.' % destination
raise FatalException(1, err)
(dest_path, file_name) = os.path.split(destination)
@@ -138,7 +138,7 @@ def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func
downloaded_size = os.stat(temp_dest).st_size
if downloaded_size != file_size:
- err = 'Size of downloaded file {} is {} bytes, it is probably damaged or incomplete' % (destination, downloaded_size)
+ err = 'Size of downloaded file {0} is {1} bytes, it is probably damaged or incomplete' % (destination, downloaded_size)
raise FatalException(1, err)
# when download is complete -> mv temp_dest destination
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-common/src/main/python/ambari_commons/logging_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/logging_utils.py b/ambari-common/src/main/python/ambari_commons/logging_utils.py
index af1390c..4ebe696 100644
--- a/ambari-common/src/main/python/ambari_commons/logging_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/logging_utils.py
@@ -20,7 +20,7 @@ limitations under the License.
_VERBOSE = False
_SILENT = False
-_DEBUG_MODE = False
+_DEBUG_MODE = 0
# terminal styles
BOLD_ON = '\033[1m'
@@ -50,6 +50,20 @@ def set_debug_mode(newVal):
global _DEBUG_MODE
_DEBUG_MODE = newVal
+def set_debug_mode_from_options(options):
+ debug_mode = 0
+ try:
+ if options.debug:
+ debug_mode = 1
+ except AttributeError:
+ pass
+ try:
+ if options.suspend_start:
+ debug_mode = debug_mode | 2
+ except AttributeError:
+ pass
+ set_debug_mode(debug_mode)
+
#
# Prints an "info" messsage.
#
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/conf/unix/ambari.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties
index e29a6e2..29bc3ac 100644
--- a/ambari-server/conf/unix/ambari.properties
+++ b/ambari-server/conf/unix/ambari.properties
@@ -22,10 +22,23 @@ security.server.keys_dir = /var/lib/ambari-server/keys
resources.dir = /var/lib/ambari-server/resources
shared.resources.dir = /usr/lib/ambari-server/lib/ambari_commons/resources
custom.action.definitions = /var/lib/ambari-server/resources/custom_action_definitions
+
+java.releases=jdk1.7,jdk1.6
+jdk1.6.desc=Oracle JDK 1.6
jdk1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-6u31-linux-x64.bin
-jce_policy1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip
+jdk1.6.dest-file=jdk-6u31-linux-x64.bin
+jdk1.6.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip
+jdk1.6.jcpol-file=jce_policy-6.zip
+jdk1.6.home=/usr/jdk64/
+jdk1.6.re=Creating (jdk.*)/jre
+jdk1.7.desc=Oracle JDK 1.7
jdk1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-linux-x64.tar.gz
-jce_policy1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip
+jdk1.7.dest-file=jdk-7u67-linux-x64.tar.gz
+jdk1.7.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip
+jdk1.7.jcpol-file=UnlimitedJCEPolicyJDK7.zip
+jdk1.7.home=/usr/jdk64/
+jdk1.7.re=(jdk.*)/jre
+
metadata.path=/var/lib/ambari-server/resources/stacks
common.services.path=/var/lib/ambari-server/resources/common-services
server.version.file=/var/lib/ambari-server/resources/version
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/conf/windows/ambari.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties
index d5c5313..3f0ed4e 100644
--- a/ambari-server/conf/windows/ambari.properties
+++ b/ambari-server/conf/windows/ambari.properties
@@ -24,7 +24,7 @@ custom.action.definitions=resources\\custom_action_definitions
#Comma-separated list of JDK versions
#java.releases=jdk1.8.20,jdk1.6.31
java.releases=jdk1.7.67
-jdk1.7.67.desc=Oracle JDK 1.7.67
+jdk1.7.67.desc=Oracle JDK 1.7
jdk1.7.67.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe
jdk1.7.67.dest-file=jdk-7u67-windows-x64.exe
jdk1.7.67.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip
http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/main/python/ambari-server-windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server-windows.py b/ambari-server/src/main/python/ambari-server-windows.py
index 9231105..b4d4878 100644
--- a/ambari-server/src/main/python/ambari-server-windows.py
+++ b/ambari-server/src/main/python/ambari-server-windows.py
@@ -26,18 +26,19 @@ import subprocess
from ambari_commons.ambari_service import AmbariService
from ambari_commons.exceptions import FatalException, NonFatalException
from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg, \
- get_verbose, set_verbose, get_silent, set_silent, get_debug_mode, set_debug_mode
+ get_verbose, set_verbose, get_silent, set_silent, get_debug_mode, set_debug_mode_from_options
from ambari_commons.os_utils import remove_file, set_open_files_limit
from ambari_commons.os_windows import SvcStatusCallback
from ambari_server import utils
-from ambari_server.dbConfiguration import DBMSConfig
+from ambari_server.dbConfiguration import DBMSConfigFactory
from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException
from ambari_server.serverConfiguration import find_jdk, get_ambari_classpath, get_ambari_properties, get_conf_dir, \
get_value_from_properties, configDefaults, DEBUG_MODE_KEY, RESOURCES_DIR_DEFAULT, RESOURCES_DIR_PROPERTY, \
- SERVER_OUT_FILE_KEY, STACK_LOCATION_DEFAULT, STACK_LOCATION_KEY, SUSPEND_START_MODE_KEY, VERBOSE_OUTPUT_KEY
-from ambari_server.serverSetup import setup, reset, is_server_running, upgrade
-from ambari_server.serverSetup_windows import SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY
+ SERVER_OUT_FILE_KEY, SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY, \
+ SUSPEND_START_MODE_KEY, VERBOSE_OUTPUT_KEY, PID_NAME, get_java_exe_path
+from ambari_server.serverSetup import setup, reset, is_server_running
+from ambari_server.serverUpgrade import upgrade
from ambari_server.setupActions import SETUP_ACTION, START_ACTION, PSTART_ACTION, STOP_ACTION, RESET_ACTION, \
STATUS_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, LDAP_SETUP_ACTION, SETUP_SECURITY_ACTION, ACTION_REQUIRE_RESTART
from ambari_server.setupSecurity import setup_ambari_krb5_jaas, setup_https, setup_ldap, setup_master_key
@@ -73,16 +74,12 @@ SERVER_START_CMD_DEBUG = \
" -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,"\
"server=y,suspend={1} -cp {0}" +\
" org.apache.ambari.server.controller.AmbariServer"
-SERVER_SEARCH_PATTERN = "org.apache.ambari.server.controller.AmbariServer"
SERVER_INIT_TIMEOUT = 5
SERVER_START_TIMEOUT = 10
-PID_NAME = "ambari-server.pid"
EXITCODE_NAME = "ambari-server.exitcode"
-SERVER_VERSION_FILE_PATH = "server.version.file"
-
# linux open-file limit
ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files'
ULIMIT_OPEN_FILES_DEFAULT = 10000
@@ -182,16 +179,9 @@ def svcstart():
pass
def server_process_main(options, scmStatus=None):
- # debug mode
- try:
- set_debug_mode(options.debug)
- except AttributeError:
- pass
-
- # stop Java process at startup?
+ # debug mode, including stop Java process at startup
try:
- global SUSPEND_START_MODE
- SUSPEND_START_MODE = options.suspend_start
+ set_debug_mode_from_options(options)
except AttributeError:
pass
@@ -236,10 +226,15 @@ def server_process_main(options, scmStatus=None):
if conf_dir.find(' ') != -1:
conf_dir = '"' + conf_dir + '"'
- java_exe = jdk_path + os.sep + configDefaults.JAVA_EXE_SUBPATH
+ java_exe = get_java_exe_path()
pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
- command_base = SERVER_START_CMD_DEBUG if (get_debug_mode() or SERVER_START_DEBUG) else SERVER_START_CMD
- suspend_mode = 'y' if SUSPEND_START_MODE else 'n'
+
+ debug_mode = get_debug_mode()
+ debug_start = (debug_mode & 1) or SERVER_START_DEBUG
+ suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
+
+ command_base = SERVER_START_CMD_DEBUG if debug_start else SERVER_START_CMD
+ suspend_mode = 'y' if suspend_start else 'n'
command = command_base.format(conf_dir, suspend_mode)
if not os.path.exists(configDefaults.PID_DIR):
os.makedirs(configDefaults.PID_DIR, 0755)
@@ -285,7 +280,8 @@ def server_process_main(options, scmStatus=None):
#Wait until the status is 'started' or a configured timeout elapses
#If the timeout has been reached, bail out with exception
def ensure_dbms_is_running(options, properties, scmStatus):
- dbms = DBMSConfig.create(options, properties, "Ambari")
+ factory = DBMSConfigFactory()
+ dbms = factory.create(options, properties, "Ambari")
if not dbms._is_jdbc_driver_installed(properties):
raise FatalException(-1, "JDBC driver is not installed. Run ambari-server setup and try again.")
@@ -317,9 +313,6 @@ def svcstop():
### Stack upgrade ###
-#def upgrade_stack(args, stack_id, repo_url=None, repo_url_os=None):
-
-
def get_resources_location(properties):
res_location = properties[RESOURCES_DIR_PROPERTY]
if res_location is None:
@@ -327,13 +320,6 @@ def get_resources_location(properties):
return res_location
# pass
-def get_stack_location(properties):
- stack_location = properties[STACK_LOCATION_KEY]
- if stack_location is None:
- stack_location = STACK_LOCATION_DEFAULT
- return stack_location
-# pass
-
#
# The Ambari Server status.
@@ -465,7 +451,6 @@ def main():
if are_cmd_line_db_args_blank(options):
options.must_set_database_options = True
- #TODO Silent is invalid here, right?
elif not are_cmd_line_db_args_valid(options):
parser.error('All database options should be set. Please see help for the options.')