You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2016/12/01 17:06:49 UTC
[04/32] ambari git commit: AMBARI-18997 ambari-server.pid might not
be created on slow hardware (dsen)
AMBARI-18997 ambari-server.pid might not be created on slow hardware (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d8ba7f1b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d8ba7f1b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d8ba7f1b
Branch: refs/heads/branch-feature-AMBARI-18456
Commit: d8ba7f1bad5f9bb96256a5ca64c0568d74b874f8
Parents: 1fb8d07
Author: Dmytro Sen <ds...@apache.org>
Authored: Wed Nov 30 13:50:37 2016 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Wed Nov 30 13:50:37 2016 +0200
----------------------------------------------------------------------
.../src/main/python/ambari_server/utils.py | 60 ++++-------------
.../src/main/python/ambari_server_main.py | 70 ++++++++++++++------
.../src/test/python/TestAmbariServer.py | 6 +-
ambari-server/src/test/python/TestUtils.py | 37 +----------
4 files changed, 66 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8ba7f1b/ambari-server/src/main/python/ambari_server/utils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/utils.py b/ambari-server/src/main/python/ambari_server/utils.py
index 5188a48..26e59ae 100644
--- a/ambari-server/src/main/python/ambari_server/utils.py
+++ b/ambari-server/src/main/python/ambari_server/utils.py
@@ -119,19 +119,19 @@ def save_pid(pid, pidfile):
pass
-def save_main_pid_ex(pids, pidfile, exclude_list=[], kill_exclude_list=False, skip_daemonize=False):
+def save_main_pid_ex(pids, pidfile, exclude_list=[], skip_daemonize=False):
"""
Save pid which is not included to exclude_list to pidfile.
- If kill_exclude_list is set to true, all processes in that
- list would be killed. It's might be useful to daemonize child process
exclude_list contains list of full executable paths which should be excluded
"""
+ pid_saved = False
try:
pfile = open(pidfile, "w")
for item in pids:
if pid_exists(item["pid"]) and (item["exe"] not in exclude_list):
pfile.write("%s\n" % item["pid"])
+ pid_saved = True
logger.info("Ambari server started with PID " + str(item["pid"]))
if pid_exists(item["pid"]) and (item["exe"] in exclude_list) and not skip_daemonize:
try:
@@ -147,67 +147,33 @@ def save_main_pid_ex(pids, pidfile, exclude_list=[], kill_exclude_list=False, sk
except Exception as e:
logger.error("Failed to close PID file " + pidfile + " due to " + str(e))
pass
+ return pid_saved
-
-def wait_for_pid(pids, server_init_timeout, occupy_port_timeout, init_web_ui_timeout,
- server_out_file, db_check_log, properties):
+def get_live_pids_count(pids):
"""
- Check pid for existence during timeout
+ Check pids for existence
"""
- from ambari_server.serverConfiguration import get_ambari_server_ui_port
- ambari_server_ui_port = int(get_ambari_server_ui_port(properties))
- server_ui_port_occupied = False
- tstart = time.time()
- pid_live = 0
- while int(time.time()-tstart) <= occupy_port_timeout and len(pids) > 0:
- sys.stdout.write('.')
- sys.stdout.flush()
- pid_live = 0
- for item in pids:
- if pid_exists(item["pid"]):
- pid_live += 1
- time.sleep(1)
+ return len([pid for pid in pids if pid_exists(pid)])
+
+def wait_for_ui_start(ambari_server_ui_port, timeout=1):
+ tstart = time.time()
+ while int(time.time()-tstart) <= timeout:
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(1)
sock.connect(('localhost', ambari_server_ui_port))
print "\nServer started listening on " + str(ambari_server_ui_port)
- server_ui_port_occupied = True
- break
+ return True
except Exception as e:
#print str(e)
pass
- if 'Database consistency check: failed' in open(server_out_file).read():
- print "\nDB configs consistency check failed. Run \"ambari-server start --skip-database-check\" to skip. " \
- "If you use this \"--skip-database-check\" option, do not make any changes to your cluster topology " \
- "or perform a cluster upgrade until you correct the database consistency issues. See " + \
- db_check_log + "for more details on the consistency issues."
- elif 'Database consistency check: warning' in open(server_out_file).read():
- print "\nDB configs consistency check found warnings. See " + db_check_log + " for more details."
- else:
- print "\nDB configs consistency check: no errors and warnings were found."
-
-
- if not server_ui_port_occupied:
- raise FatalException(1, "Server not yet listening on http port " + str(ambari_server_ui_port) +
- " after " + str(occupy_port_timeout + server_init_timeout) + " seconds. Exiting.")
-
- tstart = time.time()
- print "Waiting for 10 seconds, for server WEB UI initialization"
- while int(time.time()-tstart) <= init_web_ui_timeout and len(pids) > 0:
sys.stdout.write('.')
sys.stdout.flush()
- pid_live = 0
- for item in pids:
- if pid_exists(item["pid"]):
- pid_live += 1
time.sleep(1)
-
- return pid_live
-
+ return False
def get_symlink_path(path_to_link):
"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8ba7f1b/ambari-server/src/main/python/ambari_server_main.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server_main.py b/ambari-server/src/main/python/ambari_server_main.py
index 62f8980..b642cea 100644
--- a/ambari-server/src/main/python/ambari_server_main.py
+++ b/ambari-server/src/main/python/ambari_server_main.py
@@ -1,5 +1,3 @@
-SERVER_INIT_TIMEOUT = 5
-SERVER_START_TIMEOUT = 30
#!/usr/bin/env python
'''
@@ -33,15 +31,15 @@ from ambari_commons.os_utils import is_root, run_os_command
from ambari_server.dbConfiguration import ensure_dbms_is_running, ensure_jdbc_driver_is_installed
from ambari_server.serverConfiguration import configDefaults, find_jdk, get_ambari_properties, \
get_conf_dir, get_is_persisted, get_is_secure, get_java_exe_path, get_original_master_key, read_ambari_user, \
- get_is_active_instance, update_properties, \
+ get_is_active_instance, update_properties, get_ambari_server_ui_port, \
PID_NAME, SECURITY_KEY_ENV_VAR_NAME, SECURITY_MASTER_KEY_LOCATION, \
SETUP_OR_UPGRADE_MSG, check_database_name_property, parse_properties_file, get_missing_properties
from ambari_server.serverUtils import refresh_stack_hash
from ambari_server.setupHttps import get_fqdn
from ambari_server.setupSecurity import generate_env, \
ensure_can_start_under_current_user
-from ambari_server.utils import check_reverse_lookup, save_pid, locate_file, locate_all_file_paths, looking_for_pid, wait_for_pid, \
- save_main_pid_ex, check_exitcode
+from ambari_server.utils import check_reverse_lookup, save_pid, locate_file, locate_all_file_paths, looking_for_pid, \
+ save_main_pid_ex, check_exitcode, get_live_pids_count, wait_for_ui_start
from ambari_server.serverClassPath import ServerClassPath
logger = logging.getLogger(__name__)
@@ -103,9 +101,9 @@ SERVER_START_CMD_DEBUG_WINDOWS = "{0} " \
"-cp {3} " \
"org.apache.ambari.server.controller.AmbariServer"
-SERVER_INIT_TIMEOUT = 5 #seconds
-WEB_UI_INIT_TIME = 10 #seconds
-SERVER_START_TIMEOUT = 50 #seconds
+SERVER_START_TIMEOUT = 5 #seconds
+SERVER_START_RETRIES = 4
+WEB_UI_INIT_TIME = 50 #seconds
SERVER_PING_TIMEOUT_WINDOWS = 5
SERVER_PING_ATTEMPTS_WINDOWS = 4
@@ -117,6 +115,7 @@ EXITCODE_NAME = "ambari-server.exitcode"
CHECK_DATABASE_SKIPPED_PROPERTY = "check_database_skipped"
AMBARI_SERVER_DIE_MSG = "Ambari Server java process died with exitcode {0}. Check {1} for more information."
+AMBARI_SERVER_NOT_STARTED_MSG = "Ambari Server java process hasn't been started or can't be determined."
# linux open-file limit
ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files'
@@ -208,21 +207,48 @@ def wait_for_server_start(pidFile, scmStatus):
#wait for server process for SERVER_START_TIMEOUT seconds
sys.stdout.write('Waiting for server start...')
sys.stdout.flush()
-
- pids = looking_for_pid(SERVER_SEARCH_PATTERN, SERVER_INIT_TIMEOUT)
- found_pids = wait_for_pid(pids, SERVER_INIT_TIMEOUT, SERVER_START_TIMEOUT, WEB_UI_INIT_TIME,
- configDefaults.SERVER_OUT_FILE, configDefaults.DB_CHECK_LOG, properties)
-
- sys.stdout.write('\n')
- sys.stdout.flush()
-
- if found_pids <= 0:
+ pids = []
+ server_started = False
+ # looking_for_pid() might return partrial pid list on slow hardware
+ for i in range(1, SERVER_START_RETRIES):
+ pids = looking_for_pid(SERVER_SEARCH_PATTERN, SERVER_START_TIMEOUT)
+
+ sys.stdout.write('\n')
+ sys.stdout.flush()
+
+ if save_main_pid_ex(pids, pidFile, locate_all_file_paths('sh', '/bin') +
+ locate_all_file_paths('bash', '/bin') +
+ locate_all_file_paths('dash', '/bin'), IS_FOREGROUND):
+ server_started = True
+ break
+ else:
+ sys.stdout.write("Unable to determine server PID. Retrying...\n")
+ sys.stdout.flush()
+
+ exception = None
+ if server_started:
+ ambari_server_ui_port = get_ambari_server_ui_port(properties)
+ if not wait_for_ui_start(int(ambari_server_ui_port), WEB_UI_INIT_TIME):
+ exception = FatalException(1, "Server not yet listening on http port " + ambari_server_ui_port + \
+ " after " + str(WEB_UI_INIT_TIME) + " seconds. Exiting.")
+ elif get_live_pids_count(pids) <= 0:
exitcode = check_exitcode(os.path.join(configDefaults.PID_DIR, EXITCODE_NAME))
- raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
+ exception = FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
+ else:
+ exception = FatalException(-1, AMBARI_SERVER_NOT_STARTED_MSG)
+
+ if 'Database consistency check: failed' in open(configDefaults.SERVER_OUT_FILE).read():
+ print "DB configs consistency check failed. Run \"ambari-server start --skip-database-check\" to skip. " \
+ "If you use this \"--skip-database-check\" option, do not make any changes to your cluster topology " \
+ "or perform a cluster upgrade until you correct the database consistency issues. See " + \
+ configDefaults.DB_CHECK_LOG + " for more details on the consistency issues."
+ elif 'Database consistency check: warning' in open(configDefaults.SERVER_OUT_FILE).read():
+ print "DB configs consistency check found warnings. See " + configDefaults.DB_CHECK_LOG + " for more details."
else:
- save_main_pid_ex(pids, pidFile, locate_all_file_paths('sh', '/bin') +
- locate_all_file_paths('bash', '/bin') +
- locate_all_file_paths('dash', '/bin'), True, IS_FOREGROUND)
+ print "DB configs consistency check: no errors and warnings were found."
+
+ if exception:
+ raise exception
def server_process_main(options, scmStatus=None):
@@ -355,7 +381,7 @@ def server_process_main(options, scmStatus=None):
raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
else:
pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
- save_pid(pidJava, pidfile)
+
print "Server PID at: "+pidfile
print "Server out at: "+configDefaults.SERVER_OUT_FILE
print "Server log at: "+configDefaults.SERVER_LOG_FILE
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8ba7f1b/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 76857cd..81e1a22 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -4398,7 +4398,7 @@ class TestAmbariServer(TestCase):
@patch("sys.stdout.flush")
@patch("sys.stdout.write")
@patch("ambari_server_main.looking_for_pid")
- @patch("ambari_server_main.wait_for_pid")
+ @patch("ambari_server_main.wait_for_ui_start")
@patch("ambari_server_main.save_main_pid_ex")
@patch("ambari_server_main.check_exitcode")
@patch("os.makedirs")
@@ -4448,7 +4448,7 @@ class TestAmbariServer(TestCase):
save_master_key_method, get_master_key_location_method,
os_chown_mock, is_server_running_mock, locate_file_mock,
os_makedirs_mock, check_exitcode_mock, save_main_pid_ex_mock,
- wait_for_pid_mock, looking_for_pid_mock, stdout_write_mock, stdout_flush_mock,
+ wait_for_ui_start_mock, looking_for_pid_mock, stdout_write_mock, stdout_flush_mock,
get_is_active_instance_mock):
def reset_mocks():
@@ -4485,7 +4485,7 @@ class TestAmbariServer(TestCase):
"exe": "/test",
"cmd": "test arg"
}]
- wait_for_pid_mock.return_value = 1
+ wait_for_ui_start_mock.return_value = True
check_exitcode_mock.return_value = 0
p = Properties()
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8ba7f1b/ambari-server/src/test/python/TestUtils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestUtils.py b/ambari-server/src/test/python/TestUtils.py
index bedd75c..268d7e9 100644
--- a/ambari-server/src/test/python/TestUtils.py
+++ b/ambari-server/src/test/python/TestUtils.py
@@ -120,39 +120,6 @@ class TestUtils(TestCase):
normpath_mock.return_value = "test value"
self.assertEquals(utils.get_symlink_path("/"), "test value")
- @patch('time.time')
- @patch.object(utils, 'pid_exists')
- @patch('time.sleep')
- @patch("socket.socket")
- @patch('__builtin__.open')
- def test_wait_for_pid(self, open_mock, socket_mock, sleep_mock, pid_exists_mock, time_mock):
- from ambari_server.serverConfiguration import SSL_API, CLIENT_API_PORT_PROPERTY
- pid_exists_mock.return_value = True
- time_mock.side_effect = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51]
- s = socket_mock.return_value
- s.connect = MagicMock()
- properties = FakeProperties({
- SSL_API: "false",
- CLIENT_API_PORT_PROPERTY: "8080"
- })
- out = StringIO.StringIO()
- sys.stdout = out
- live_pids = utils.wait_for_pid([
- {"pid": "111",
- "exe": "",
- "cmd": ""
- },
- {"pid": "222",
- "exe": "",
- "cmd": ""
- },
- ], 5, 40, 10, '', '', properties)
- self.assertEqual(".\nServer started listening on 8080\n\nDB configs consistency check: no errors and warnings were "
- "found.\nWaiting for 10 seconds, for server WEB UI initialization\n........", out.getvalue())
- sys.stdout = sys.__stdout__
-
- self.assertEquals(2, live_pids)
-
@patch.object(utils, 'pid_exists')
@patch('__builtin__.open')
@patch('os.kill')
@@ -179,7 +146,7 @@ class TestUtils(TestCase):
"exe": "/exe2",
"cmd": ""
},
- ], "/pidfile", ["/exe1"], True)
+ ], "/pidfile", ["/exe1"])
self.assertEquals(open_mock.call_count, 1)
self.assertEquals(pid_exists_mock.call_count, 4)
self.assertEquals(kill_mock.call_count, 1)
@@ -257,4 +224,4 @@ class FakeProperties(object):
self.prop_map = prop_map
def get_property(self, prop_name):
- return self.prop_map[prop_name]
\ No newline at end of file
+ return self.prop_map[prop_name]