You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by fb...@apache.org on 2015/01/24 02:37:12 UTC

[8/8] ambari git commit: AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4

AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 1.4

+Further decomposing ambari-server.py: Moved setup, reset and upgrade into their own files
+Made them OS-independent - now the same routines are being used in Windows too (save upgrade, which is not active in Windows yet)
+Merged the OS-dependent code for JDBC & JDK setup, database interface creation, firewall checks
+Refactored the database setup: split the supported DBMSes into distinct objects
+Replaced recursive calls in the user input routines


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/49955a35
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/49955a35
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/49955a35

Branch: refs/heads/trunk
Commit: 49955a35971f723dcc8d4acee57fbd9d6bec58f5
Parents: ccc58c1
Author: Florian Barca <fb...@hortonworks.com>
Authored: Fri Jan 23 17:36:55 2015 -0800
Committer: Florian Barca <fb...@hortonworks.com>
Committed: Fri Jan 23 17:36:55 2015 -0800

----------------------------------------------------------------------
 .../src/main/python/ambari_commons/firewall.py  |   90 +-
 .../main/python/ambari_commons/inet_utils.py    |    6 +-
 .../main/python/ambari_commons/logging_utils.py |   16 +-
 ambari-server/conf/unix/ambari.properties       |   17 +-
 ambari-server/conf/windows/ambari.properties    |    2 +-
 .../src/main/python/ambari-server-windows.py    |   51 +-
 ambari-server/src/main/python/ambari-server.py  | 2789 +---------------
 .../python/ambari_server/dbConfiguration.py     |  432 ++-
 .../ambari_server/dbConfiguration_linux.py      | 1300 ++++----
 .../ambari_server/dbConfiguration_windows.py    |  298 +-
 .../python/ambari_server/serverConfiguration.py |  449 ++-
 .../main/python/ambari_server/serverSetup.py    | 1075 ++++--
 .../python/ambari_server/serverSetup_linux.py   |  793 -----
 .../python/ambari_server/serverSetup_windows.py |  300 --
 .../main/python/ambari_server/serverUpgrade.py  |  305 ++
 .../main/python/ambari_server/setupActions.py   |    4 +-
 .../main/python/ambari_server/setupSecurity.py  |  275 +-
 .../src/main/python/ambari_server/userInput.py  |   29 +-
 .../custom_actions/scripts/check_host.py        |    2 +-
 .../2.1/hooks/before-ANY/scripts/setup_jdk.py   |    4 +-
 .../package/scripts/mapred_service_check.py     |    4 +-
 .../src/test/python/TestAmbariServer.py         | 3081 ++++++++++--------
 ambari-server/src/test/python/TestOSCheck.py    |    7 +-
 23 files changed, 4782 insertions(+), 6547 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-common/src/main/python/ambari_commons/firewall.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/firewall.py b/ambari-common/src/main/python/ambari_commons/firewall.py
index 76d9890..311e6d5 100644
--- a/ambari-common/src/main/python/ambari_commons/firewall.py
+++ b/ambari-common/src/main/python/ambari_commons/firewall.py
@@ -18,9 +18,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
+import subprocess
+import shlex
 from ambari_commons import OSCheck, OSConst
+from ambari_commons.logging_utils import print_warning_msg
+from ambari_commons.os_family_impl import OsFamilyImpl
 from ambari_commons.os_utils import run_os_command
 
+
 class Firewall(object):
   def __init__(self):
     # OS info
@@ -29,6 +34,16 @@ class Firewall(object):
     self.OS_FAMILY = OSCheck.get_os_family()
 
   def getFirewallObject(self):
+    pass
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class FirewallWindows(Firewall):
+  def getFirewallObject(self):
+    return WindowsFirewallChecks()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class FirewallLinux(Firewall):
+  def getFirewallObject(self):
     if self.OS_TYPE == OSConst.OS_UBUNTU:
       return UbuntuFirewallChecks()
     elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18:
@@ -47,23 +62,31 @@ class FirewallChecks(object):
     self.returncode = None
     self.stdoutdata = None
     self.stderrdata = None
+    # stdout message
+    self.MESSAGE_CHECK_FIREWALL = 'Checking iptables...'
 
   def get_command(self):
     return "%s %s %s" % (self.SERVICE_CMD, self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD)
 
-  def check_result(self, retcode, out, err):
+  def check_result(self):
     result = False
-    if retcode == 3:
+    if self.returncode == 3:
       result = False
-    elif retcode == 0:
-      if "Table: filter" in out:
+    elif self.returncode == 0:
+      if "Table: filter" in self.stdoutdata:
         result = True
     return result
 
+  def run_command(self):
+    retcode, out, err = run_os_command(self.get_command())
+    self.returncode = retcode
+    self.stdoutdata = out
+    self.stderrdata = err
+
   def check_iptables(self):
     try:
-      self.returncode, self.stdoutdata, self.stderrdata = run_os_command(self.get_command())
-      return self.check_result(self.returncode, self.stdoutdata, self.stderrdata)
+      self.run_command()
+      return self.check_result()
     except OSError:
       return False
 
@@ -76,13 +99,13 @@ class UbuntuFirewallChecks(FirewallChecks):
   def get_command(self):
     return "%s %s" % (self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD)
 
-  def check_result(self, retcode, out, err):
+  def check_result(self):
     # On ubuntu, the status command returns 0 whether running or not
     result = False
-    if retcode == 0:
-      if "Status: inactive" in out:
+    if self.returncode == 0:
+      if "Status: inactive" in self.stdoutdata:
         result = False
-      elif "Status: active" in out:
+      elif "Status: active" in self.stdoutdata:
         result = True
     return result
 
@@ -93,10 +116,10 @@ class Fedora18FirewallChecks(FirewallChecks):
   def get_command(self):
     return "systemctl is-active %s" % (self.FIREWALL_SERVICE_NAME)
 
-  def check_result(self, retcode, out, err):
+  def check_result(self):
     result = False
-    if retcode == 0:
-      if "active" in out:
+    if self.returncode == 0:
+      if "active" in self.stdoutdata:
         result = True
     return result
 
@@ -108,11 +131,44 @@ class SuseFirewallChecks(FirewallChecks):
   def get_command(self):
     return "%s %s" % (self.FIREWALL_SERVICE_NAME, self.SERVICE_SUBCMD)
 
-  def check_result(self, retcode, out, err):
+  def check_result(self):
     result = False
-    if retcode == 0:
-      if "SuSEfirewall2 not active" in out:
+    if self.returncode == 0:
+      if "SuSEfirewall2 not active" in self.stdoutdata:
         result = False
-      elif "### iptables" in out:
+      elif "### iptables" in self.stdoutdata:
         result = True
     return result
+
+
+class WindowsFirewallChecks(FirewallChecks):
+  def __init__(self):
+    super(WindowsFirewallChecks, self).__init__()
+    self.MESSAGE_CHECK_FIREWALL = 'Checking firewall status...'
+
+  def run_command(self):
+    from ambari_commons.os_windows import run_powershell_script, CHECK_FIREWALL_SCRIPT
+
+    retcode, out, err = run_powershell_script(CHECK_FIREWALL_SCRIPT)
+    self.returncode = retcode
+    self.stdoutdata = out
+    self.stderrdata = err
+
+  def check_result(self):
+    if self.returncode != 0:
+      print_warning_msg("Unable to check firewall status:{0}".format(self.stderrdata))
+      return False
+    profiles_status = [i for i in self.stdoutdata.split("\n") if not i == ""]
+    if "1" in profiles_status:
+      enabled_profiles = []
+      if profiles_status[0] == "1":
+        enabled_profiles.append("DomainProfile")
+      if profiles_status[1] == "1":
+        enabled_profiles.append("StandardProfile")
+      if profiles_status[2] == "1":
+        enabled_profiles.append("PublicProfile")
+      print_warning_msg(
+        "Following firewall profiles are enabled:{0}. Make sure that the firewall is properly configured.".format(
+          ",".join(enabled_profiles)))
+      return False
+    return True

http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-common/src/main/python/ambari_commons/inet_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py b/ambari-common/src/main/python/ambari_commons/inet_utils.py
index 2a54cb6..5c9275c 100644
--- a/ambari-common/src/main/python/ambari_commons/inet_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py
@@ -56,7 +56,7 @@ def find_range_components(meta):
     if len(range_comp1) > 1:
       range_comp2 = range_comp1[0].split(' ') #split away the "bytes" prefix
       if len(range_comp2) == 0:
-        raise FatalException(12, 'Malformed Content-Range response header: "{}".' % hdr_range)
+        raise FatalException(12, 'Malformed Content-Range response header: "{0}".' % hdr_range)
       range_comp3 = range_comp2[1].split('-')
       seek_pos = int(range_comp3[0])
       if range_comp1[1] != '*': #'*' == unknown length
@@ -77,7 +77,7 @@ def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func
 
   if os.path.exists(destination) and not os.path.isfile(destination):
     #Directory specified as target? Must be a mistake. Bail out, don't assume anything.
-    err = 'Download target {} is a directory.' % destination
+    err = 'Download target {0} is a directory.' % destination
     raise FatalException(1, err)
 
   (dest_path, file_name) = os.path.split(destination)
@@ -138,7 +138,7 @@ def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func
 
   downloaded_size = os.stat(temp_dest).st_size
   if downloaded_size != file_size:
-    err = 'Size of downloaded file {} is {} bytes, it is probably damaged or incomplete' % (destination, downloaded_size)
+    err = 'Size of downloaded file {0} is {1} bytes, it is probably damaged or incomplete' % (destination, downloaded_size)
     raise FatalException(1, err)
 
   # when download is complete -> mv temp_dest destination

http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-common/src/main/python/ambari_commons/logging_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/logging_utils.py b/ambari-common/src/main/python/ambari_commons/logging_utils.py
index af1390c..4ebe696 100644
--- a/ambari-common/src/main/python/ambari_commons/logging_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/logging_utils.py
@@ -20,7 +20,7 @@ limitations under the License.
 
 _VERBOSE = False
 _SILENT = False
-_DEBUG_MODE = False
+_DEBUG_MODE = 0
 
 # terminal styles
 BOLD_ON = '\033[1m'
@@ -50,6 +50,20 @@ def set_debug_mode(newVal):
   global _DEBUG_MODE
   _DEBUG_MODE = newVal
 
+def set_debug_mode_from_options(options):
+  debug_mode = 0
+  try:
+    if options.debug:
+      debug_mode = 1
+  except AttributeError:
+    pass
+  try:
+    if options.suspend_start:
+      debug_mode = debug_mode | 2
+  except AttributeError:
+    pass
+  set_debug_mode(debug_mode)
+
 #
 # Prints an "info" messsage.
 #

http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/conf/unix/ambari.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties
index e29a6e2..29bc3ac 100644
--- a/ambari-server/conf/unix/ambari.properties
+++ b/ambari-server/conf/unix/ambari.properties
@@ -22,10 +22,23 @@ security.server.keys_dir = /var/lib/ambari-server/keys
 resources.dir = /var/lib/ambari-server/resources
 shared.resources.dir = /usr/lib/ambari-server/lib/ambari_commons/resources
 custom.action.definitions = /var/lib/ambari-server/resources/custom_action_definitions
+
+java.releases=jdk1.7,jdk1.6
+jdk1.6.desc=Oracle JDK 1.6
 jdk1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-6u31-linux-x64.bin
-jce_policy1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip
+jdk1.6.dest-file=jdk-6u31-linux-x64.bin
+jdk1.6.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zip
+jdk1.6.jcpol-file=jce_policy-6.zip
+jdk1.6.home=/usr/jdk64/
+jdk1.6.re=Creating (jdk.*)/jre
+jdk1.7.desc=Oracle JDK 1.7
 jdk1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-linux-x64.tar.gz
-jce_policy1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip
+jdk1.7.dest-file=jdk-7u67-linux-x64.tar.gz
+jdk1.7.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip
+jdk1.7.jcpol-file=UnlimitedJCEPolicyJDK7.zip
+jdk1.7.home=/usr/jdk64/
+jdk1.7.re=(jdk.*)/jre
+
 metadata.path=/var/lib/ambari-server/resources/stacks
 common.services.path=/var/lib/ambari-server/resources/common-services
 server.version.file=/var/lib/ambari-server/resources/version

http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/conf/windows/ambari.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties
index d5c5313..3f0ed4e 100644
--- a/ambari-server/conf/windows/ambari.properties
+++ b/ambari-server/conf/windows/ambari.properties
@@ -24,7 +24,7 @@ custom.action.definitions=resources\\custom_action_definitions
 #Comma-separated list of JDK versions
 #java.releases=jdk1.8.20,jdk1.6.31
 java.releases=jdk1.7.67
-jdk1.7.67.desc=Oracle JDK 1.7.67
+jdk1.7.67.desc=Oracle JDK 1.7
 jdk1.7.67.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-windows-x64.exe
 jdk1.7.67.dest-file=jdk-7u67-windows-x64.exe
 jdk1.7.67.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip

http://git-wip-us.apache.org/repos/asf/ambari/blob/49955a35/ambari-server/src/main/python/ambari-server-windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server-windows.py b/ambari-server/src/main/python/ambari-server-windows.py
index 9231105..b4d4878 100644
--- a/ambari-server/src/main/python/ambari-server-windows.py
+++ b/ambari-server/src/main/python/ambari-server-windows.py
@@ -26,18 +26,19 @@ import subprocess
 from ambari_commons.ambari_service import AmbariService
 from ambari_commons.exceptions import FatalException, NonFatalException
 from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg, \
-  get_verbose, set_verbose, get_silent, set_silent, get_debug_mode, set_debug_mode
+  get_verbose, set_verbose, get_silent, set_silent, get_debug_mode, set_debug_mode_from_options
 from ambari_commons.os_utils import remove_file, set_open_files_limit
 from ambari_commons.os_windows import SvcStatusCallback
 
 from ambari_server import utils
-from ambari_server.dbConfiguration import DBMSConfig
+from ambari_server.dbConfiguration import DBMSConfigFactory
 from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException
 from ambari_server.serverConfiguration import find_jdk, get_ambari_classpath, get_ambari_properties, get_conf_dir, \
   get_value_from_properties, configDefaults, DEBUG_MODE_KEY, RESOURCES_DIR_DEFAULT, RESOURCES_DIR_PROPERTY, \
-  SERVER_OUT_FILE_KEY, STACK_LOCATION_DEFAULT, STACK_LOCATION_KEY, SUSPEND_START_MODE_KEY, VERBOSE_OUTPUT_KEY
-from ambari_server.serverSetup import setup, reset, is_server_running, upgrade
-from ambari_server.serverSetup_windows import SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY
+  SERVER_OUT_FILE_KEY, SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY, \
+  SUSPEND_START_MODE_KEY, VERBOSE_OUTPUT_KEY, PID_NAME, get_java_exe_path
+from ambari_server.serverSetup import setup, reset, is_server_running
+from ambari_server.serverUpgrade import upgrade
 from ambari_server.setupActions import SETUP_ACTION, START_ACTION, PSTART_ACTION, STOP_ACTION, RESET_ACTION, \
   STATUS_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, LDAP_SETUP_ACTION, SETUP_SECURITY_ACTION, ACTION_REQUIRE_RESTART
 from ambari_server.setupSecurity import setup_ambari_krb5_jaas, setup_https, setup_ldap, setup_master_key
@@ -73,16 +74,12 @@ SERVER_START_CMD_DEBUG = \
                        " -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,"\
                        "server=y,suspend={1} -cp {0}" +\
                        " org.apache.ambari.server.controller.AmbariServer"
-SERVER_SEARCH_PATTERN = "org.apache.ambari.server.controller.AmbariServer"
 
 SERVER_INIT_TIMEOUT = 5
 SERVER_START_TIMEOUT = 10
 
-PID_NAME = "ambari-server.pid"
 EXITCODE_NAME = "ambari-server.exitcode"
 
-SERVER_VERSION_FILE_PATH = "server.version.file"
-
 # linux open-file limit
 ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files'
 ULIMIT_OPEN_FILES_DEFAULT = 10000
@@ -182,16 +179,9 @@ def svcstart():
   pass
 
 def server_process_main(options, scmStatus=None):
-  # debug mode
-  try:
-    set_debug_mode(options.debug)
-  except AttributeError:
-    pass
-
-  # stop Java process at startup?
+  # debug mode, including stop Java process at startup
   try:
-    global SUSPEND_START_MODE
-    SUSPEND_START_MODE = options.suspend_start
+    set_debug_mode_from_options(options)
   except AttributeError:
     pass
 
@@ -236,10 +226,15 @@ def server_process_main(options, scmStatus=None):
   if conf_dir.find(' ') != -1:
     conf_dir = '"' + conf_dir + '"'
 
-  java_exe = jdk_path + os.sep + configDefaults.JAVA_EXE_SUBPATH
+  java_exe = get_java_exe_path()
   pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
-  command_base = SERVER_START_CMD_DEBUG if (get_debug_mode() or SERVER_START_DEBUG) else SERVER_START_CMD
-  suspend_mode = 'y' if SUSPEND_START_MODE else 'n'
+
+  debug_mode = get_debug_mode()
+  debug_start = (debug_mode & 1) or SERVER_START_DEBUG
+  suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
+
+  command_base = SERVER_START_CMD_DEBUG if debug_start else SERVER_START_CMD
+  suspend_mode = 'y' if suspend_start else 'n'
   command = command_base.format(conf_dir, suspend_mode)
   if not os.path.exists(configDefaults.PID_DIR):
     os.makedirs(configDefaults.PID_DIR, 0755)
@@ -285,7 +280,8 @@ def server_process_main(options, scmStatus=None):
 #Wait until the status is 'started' or a configured timeout elapses
 #If the timeout has been reached, bail out with exception
 def ensure_dbms_is_running(options, properties, scmStatus):
-  dbms = DBMSConfig.create(options, properties, "Ambari")
+  factory = DBMSConfigFactory()
+  dbms = factory.create(options, properties, "Ambari")
   if not dbms._is_jdbc_driver_installed(properties):
     raise FatalException(-1, "JDBC driver is not installed. Run ambari-server setup and try again.")
 
@@ -317,9 +313,6 @@ def svcstop():
 
 ### Stack upgrade ###
 
-#def upgrade_stack(args, stack_id, repo_url=None, repo_url_os=None):
-
-
 def get_resources_location(properties):
   res_location = properties[RESOURCES_DIR_PROPERTY]
   if res_location is None:
@@ -327,13 +320,6 @@ def get_resources_location(properties):
   return res_location
 #  pass
 
-def get_stack_location(properties):
-  stack_location = properties[STACK_LOCATION_KEY]
-  if stack_location is None:
-    stack_location = STACK_LOCATION_DEFAULT
-  return stack_location
-#  pass
-
 
 #
 # The Ambari Server status.
@@ -465,7 +451,6 @@ def main():
 
   if are_cmd_line_db_args_blank(options):
     options.must_set_database_options = True
-    #TODO Silent is invalid here, right?
 
   elif not are_cmd_line_db_args_valid(options):
     parser.error('All database options should be set. Please see help for the options.')