You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/10/03 17:51:36 UTC
[1/6] ambari git commit: AMBARI-18505. Ambari Status commands should
enforce a timeout < heartbeat interval (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-dev-patch-upgrade 806723e0e -> e6d8cd119
AMBARI-18505. Ambari Status commands should enforce a timeout < heartbeat interval (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dfa16136
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dfa16136
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dfa16136
Branch: refs/heads/branch-dev-patch-upgrade
Commit: dfa16136d0ceaba1e2fdb67fbc4d9dc3e3ec49f5
Parents: 324107d
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Oct 3 12:28:46 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Oct 3 12:28:46 2016 +0300
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent.ini | 1 +
.../src/main/python/ambari_agent/ActionQueue.py | 22 ++++++++++++++++-
.../ambari_agent/PythonReflectiveExecutor.py | 25 +++++++++++++++-----
.../test/python/ambari_agent/TestActionQueue.py | 3 ++-
4 files changed, 43 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/dfa16136/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini
index 914e09a..1c39c24 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -32,6 +32,7 @@ tolerate_download_failures=true
run_as_user=root
parallel_execution=0
alert_grace_period=5
+status_command_timeout=2
alert_kinit_timeout=14400000
system_resource_overrides=/etc/resource_overrides
; memory_threshold_soft_mb=400
http://git-wip-us.apache.org/repos/asf/ambari/blob/dfa16136/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
index f104939..86918e5 100644
--- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
+++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
@@ -29,6 +29,7 @@ import time
import signal
from AgentException import AgentException
+from PythonReflectiveExecutor import PythonReflectiveExecutor
from LiveStatus import LiveStatus
from ActualConfigHandler import ActualConfigHandler
from CommandStatusDict import CommandStatusDict
@@ -82,9 +83,11 @@ class ActionQueue(threading.Thread):
self.controller = controller
self.configTags = {}
self._stop = threading.Event()
+ self.hangingStatusCommands = {}
self.tmpdir = config.get('agent', 'prefix')
self.customServiceOrchestrator = CustomServiceOrchestrator(config, controller)
self.parallel_execution = config.get_parallel_exec_option()
+ self.status_command_timeout = int(self.config.get('agent', 'status_command_timeout', 5))
if self.parallel_execution == 1:
logger.info("Parallel execution is enabled, will execute agent commands in parallel")
@@ -225,7 +228,24 @@ class ActionQueue(threading.Thread):
if self.controller.recovery_manager.enabled():
self.controller.recovery_manager.stop_execution_command()
elif commandType == self.STATUS_COMMAND:
- self.execute_status_command(command)
+ component_name = command['componentName']
+
+ if component_name in self.hangingStatusCommands and not self.hangingStatusCommands[component_name].isAlive():
+ del self.hangingStatusCommands[component_name]
+
+ if not component_name in self.hangingStatusCommands:
+ thread = threading.Thread(target = self.execute_status_command, args = (command,))
+ thread.daemon = True # hanging status commands should not be prevent ambari-agent from stopping
+ thread.start()
+ thread.join(timeout=self.status_command_timeout)
+
+ if thread.isAlive():
+ # Force context to reset to normal. By context we mean sys.path, imports, logger setting, etc. They are set by specific status command, and are not relevant to ambari-agent.
+ PythonReflectiveExecutor.last_context.revert()
+ logger.warn("Command {0} for {1} is running for more than {2} seconds. Skipping it for current pack of status commands.".format(commandType, component_name, self.status_command_timeout))
+ self.hangingStatusCommands[component_name] = thread
+ else:
+ logger.info("Not running {0} for {1}, because previous one is still running.".format(commandType, component_name))
else:
logger.error("Unrecognized command " + pprint.pformat(command))
except Exception:
http://git-wip-us.apache.org/repos/asf/ambari/blob/dfa16136/ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py b/ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py
index 655b2fc..b476671 100644
--- a/ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py
+++ b/ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py
@@ -53,7 +53,9 @@ class PythonReflectiveExecutor(PythonExecutor):
returncode = 1
try:
- with PythonContext(script_dir, pythonCommand):
+ current_context = PythonContext(script_dir, pythonCommand)
+ PythonReflectiveExecutor.last_context = current_context
+ with current_context:
imp.load_source('__main__', script)
except SystemExit as e:
returncode = e.code
@@ -62,7 +64,10 @@ class PythonReflectiveExecutor(PythonExecutor):
except (ClientComponentHasNoStatus, ComponentIsNotRunning):
logger.debug("Reflective command failed with exception:", exc_info=1)
except Exception:
- logger.info("Reflective command failed with exception:", exc_info=1)
+ if current_context.is_forced_revert:
+ logger.info("Hanging status command finished its execution")
+ else:
+ logger.info("Reflective command failed with exception:", exc_info=1)
else:
returncode = 0
@@ -76,6 +81,8 @@ class PythonContext:
def __init__(self, script_dir, pythonCommand):
self.script_dir = script_dir
self.pythonCommand = pythonCommand
+ self.is_reverted = False
+ self.is_forced_revert = False
def __enter__(self):
self.old_sys_path = copy.copy(sys.path)
@@ -88,12 +95,18 @@ class PythonContext:
sys.argv = self.pythonCommand[1:]
def __exit__(self, exc_type, exc_val, exc_tb):
- sys.path = self.old_sys_path
- sys.argv = self.old_agv
- logging.disable(self.old_logging_disable)
- self.revert_sys_modules(self.old_sys_modules)
+ self.revert(is_forced_revert=False)
return False
+ def revert(self, is_forced_revert=True):
+ if not self.is_reverted:
+ self.is_forced_revert = is_forced_revert
+ self.is_reverted = True
+ sys.path = self.old_sys_path
+ sys.argv = self.old_agv
+ logging.disable(self.old_logging_disable)
+ self.revert_sys_modules(self.old_sys_modules)
+
def revert_sys_modules(self, value):
sys.modules.update(value)
http://git-wip-us.apache.org/repos/asf/ambari/blob/dfa16136/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
index 7d04d42..32773b8 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
@@ -225,6 +225,7 @@ class TestActionQueue(TestCase):
retryable_command = {
'commandType': 'EXECUTION_COMMAND',
'role': 'NAMENODE',
+ 'componentName': 'NAMENODE',
'roleCommand': 'INSTALL',
'commandId': '1-1',
'taskId': 19,
@@ -322,6 +323,7 @@ class TestActionQueue(TestCase):
}
status_command = {
'commandType' : ActionQueue.STATUS_COMMAND,
+ 'componentName': 'NAMENODE'
}
wrong_command = {
'commandType' : "SOME_WRONG_COMMAND",
@@ -1126,7 +1128,6 @@ class TestActionQueue(TestCase):
self.assertTrue(runCommand_mock.called)
self.assertEqual(2, runCommand_mock.call_count)
self.assertEqual(1, sleep_mock.call_count)
- sleep_mock.assert_has_calls([call(1)], False)
runCommand_mock.assert_has_calls([
call(command, os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'errors-19.txt', override_output_files=True, retry=False),
[5/6] ambari git commit: AMBARI-18514. Graph details popup: file
formats dropdown isn't collapsed on text link click (alexantonenko)
Posted by nc...@apache.org.
AMBARI-18514. Graph details popup: file formats dropdown isn't collapsed on text link click (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f2bcbbe7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f2bcbbe7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f2bcbbe7
Branch: refs/heads/branch-dev-patch-upgrade
Commit: f2bcbbe70008303afdefd7c218b4954127ebe978
Parents: 7b15e38
Author: Alex Antonenko <hi...@gmail.com>
Authored: Mon Oct 3 15:34:23 2016 +0300
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Mon Oct 3 20:20:56 2016 +0300
----------------------------------------------------------------------
ambari-web/app/views/common/chart/linear_time.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f2bcbbe7/ambari-web/app/views/common/chart/linear_time.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/chart/linear_time.js b/ambari-web/app/views/common/chart/linear_time.js
index 80a30d5..8eaf6f3 100644
--- a/ambari-web/app/views/common/chart/linear_time.js
+++ b/ambari-web/app/views/common/chart/linear_time.js
@@ -974,7 +974,7 @@ App.ChartLinearTimeView = Ember.View.extend(App.ExportMetricsMixin, {
title: Em.I18n.t('common.export')
});
this.$().closest('.modal').on('click', function (event) {
- if (!($(event.target).is('.corner-icon, .icon-save, .export-graph-list-container, .export-graph-list-container *'))) {
+ if (!($(event.target).is('.corner-icon, .corner-icon span, .icon-save, .export-graph-list-container, .export-graph-list-container *'))) {
popupBody.set('isExportMenuHidden', true);
}
});
[3/6] ambari git commit: AMBARI-18488. Delete group button always
shows tooltip "Cannot Delete Group" (alexantonenko)
Posted by nc...@apache.org.
AMBARI-18488. Delete group button always shows tooltip "Cannot Delete Group" (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/30eadbf0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/30eadbf0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/30eadbf0
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 30eadbf07fe601931e9029c4a7007218b62194ad
Parents: d6d5df5
Author: Alex Antonenko <hi...@gmail.com>
Authored: Mon Oct 3 15:22:34 2016 +0300
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Mon Oct 3 15:22:57 2016 +0300
----------------------------------------------------------------------
.../src/main/resources/ui/admin-web/app/views/groups/edit.html | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/30eadbf0/ambari-admin/src/main/resources/ui/admin-web/app/views/groups/edit.html
----------------------------------------------------------------------
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/groups/edit.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/groups/edit.html
index 8271432..78a6884 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/views/groups/edit.html
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/groups/edit.html
@@ -24,7 +24,7 @@
<div class="pull-right top-margin-4">
<div ng-switch="group.ldap_group">
<button ng-switch-when="true" class="btn disabled deletegroup-btn deleteuser-btn" tooltip="{{'common.cannotDelete' | translate: '{term: constants.group}'}}">{{'common.delete' | translate: '{term: constants.group}'}}</button>
- <button ng-switch-when="false" class="btn btn-danger deletegroup-btn" ng-click="deleteGroup(group)" tooltip="{{'common.cannotDelete' | translate: '{term: constants.group}'}}">{{'common.delete' | translate: '{term: constants.group}'}}</button>
+ <button ng-switch-when="false" class="btn btn-danger deletegroup-btn" ng-click="deleteGroup(group)">{{'common.delete' | translate: '{term: constants.group}'}}</button>
</div>
</div>
@@ -92,4 +92,4 @@
<div class="alert alert-info hide-soft" ng-class="{'visible' : !privileges}">{{'common.alerts.noPrivilegesDescription' | translate: '{term: constants.group.toLowerCase()}'}}</div>
</div>
</div>
-</form>
\ No newline at end of file
+</form>
[4/6] ambari git commit: AMBARI-18499 Ability to remove broken
symbolic links via HostCleanup.py (dili)
Posted by nc...@apache.org.
AMBARI-18499 Ability to remove broken symbolic links via HostCleanup.py (dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7b15e388
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7b15e388
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7b15e388
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 7b15e388c37cd2cabf224568760ab127d07412ca
Parents: 30eadbf
Author: Di Li <di...@apache.org>
Authored: Mon Oct 3 11:44:52 2016 -0400
Committer: Di Li <di...@apache.org>
Committed: Mon Oct 3 11:44:52 2016 -0400
----------------------------------------------------------------------
ambari-agent/src/main/python/ambari_agent/HostCleanup.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b15e388/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index e9ae8e5..a3c72e6 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -405,13 +405,13 @@ class HostCleanup:
def do_erase_files_silent(self, pathList):
if pathList:
for path in pathList:
- if path and os.path.exists(path):
+ if path and ( os.path.exists(path) or os.path.islink(path) ):
try:
os.remove(path)
except:
- logger.warn("Failed to delete file: " + path + ", error: " + str(sys.exc_info()[0]))
+ logger.warn("Failed to delete file: {0}, error: {1}".format(path, str(sys.exc_info()[0])))
else:
- logger.info("File doesn't exists: " + path)
+ logger.info("File doesn't exists: {0}".format(path))
return 0
def do_delete_group(self):
[6/6] ambari git commit: Merge branch 'trunk' into
branch-dev-patch-upgrade
Posted by nc...@apache.org.
Merge branch 'trunk' into branch-dev-patch-upgrade
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e6d8cd11
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e6d8cd11
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e6d8cd11
Branch: refs/heads/branch-dev-patch-upgrade
Commit: e6d8cd119f9aaecbc6de33b5784d4cf2906ad53b
Parents: 806723e f2bcbbe
Author: Nate Cole <nc...@hortonworks.com>
Authored: Mon Oct 3 13:51:21 2016 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Mon Oct 3 13:51:21 2016 -0400
----------------------------------------------------------------------
.../ui/admin-web/app/views/groups/edit.html | 4 ++--
ambari-agent/conf/unix/ambari-agent.ini | 1 +
.../src/main/python/ambari_agent/ActionQueue.py | 22 ++++++++++++++++-
.../src/main/python/ambari_agent/HostCleanup.py | 6 ++---
.../ambari_agent/PythonReflectiveExecutor.py | 25 +++++++++++++++-----
.../test/python/ambari_agent/TestActionQueue.py | 3 ++-
.../app/views/common/chart/linear_time.js | 2 +-
7 files changed, 49 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[2/6] ambari git commit: AMBARI-18505. Ambari Status commands should
enforce a timeout < heartbeat interval (aonishuk)
Posted by nc...@apache.org.
AMBARI-18505. Ambari Status commands should enforce a timeout < heartbeat interval (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d6d5df5c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d6d5df5c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d6d5df5c
Branch: refs/heads/branch-dev-patch-upgrade
Commit: d6d5df5c47f28916fbeec1d019f07d615a4e8d4d
Parents: dfa1613
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Oct 3 12:39:28 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Oct 3 12:39:28 2016 +0300
----------------------------------------------------------------------
ambari-agent/src/main/python/ambari_agent/ActionQueue.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d6d5df5c/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
index 86918e5..c03ee4f 100644
--- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
+++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
@@ -87,7 +87,7 @@ class ActionQueue(threading.Thread):
self.tmpdir = config.get('agent', 'prefix')
self.customServiceOrchestrator = CustomServiceOrchestrator(config, controller)
self.parallel_execution = config.get_parallel_exec_option()
- self.status_command_timeout = int(self.config.get('agent', 'status_command_timeout', 5))
+ self.status_command_timeout = int(self.config.get('agent', 'status_command_timeout', 2))
if self.parallel_execution == 1:
logger.info("Parallel execution is enabled, will execute agent commands in parallel")