You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2018/07/01 12:28:05 UTC
[ambari] branch trunk updated: [AMBARI-24228] Agent-side
command-*.json files should optionally be deleted when no longer needed by
the command
This is an automated email from the ASF dual-hosted git repository.
rlevas pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new fb6bde2 [AMBARI-24228] Agent-side command-*.json files should optionally be deleted when no longer needed by the command
fb6bde2 is described below
commit fb6bde26d2a874e6d04b5ced5d421fba8d9eb951
Author: Robert Levas <rl...@users.noreply.github.com>
AuthorDate: Sun Jul 1 08:28:01 2018 -0400
[AMBARI-24228] Agent-side command-*.json files should optionally be deleted when no longer needed by the command
* [AMBARI-24228] Agent-side command-*.json files should optionally be deleted when no longer needed by the command
* [AMBARI-24228] Agent-side command-*.json files should optionally be deleted when no longer needed by the command
* [AMBARI-24228] Agent-side command-*.json files should optionally be deleted when no longer needed by the command
* Revert "[AMBARI-24228] Agent-side command-*.json files should optionally be deleted when no longer needed by the command"
This reverts commit 283d3d2265e2ee6f8aefd85ab46bfd843f25a670.
* [AMBARI-24228] Agent-side command-*.json files should optionally be deleted when no longer needed by the command
---
.../src/main/python/ambari_agent/AmbariConfig.py | 49 +++++++++++++++++
.../ambari_agent/CustomServiceOrchestrator.py | 62 ++++++++++++++++++++++
.../test/python/ambari_agent/TestAmbariConfig.py | 29 ++++++++++
3 files changed, 140 insertions(+)
diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
index 68cb593..9a7a41a 100644
--- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
+++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
@@ -82,6 +82,7 @@ log_command_executes = 0
class AmbariConfig:
TWO_WAY_SSL_PROPERTY = "security.server.two_way_ssl"
+ COMMAND_FILE_RETENTION_POLICY_PROPERTY = 'command_file_retention_policy'
AMBARI_PROPERTIES_CATEGORY = 'agentConfig'
SERVER_CONNECTION_INFO = "{0}/connection_info"
CONNECTION_PROTOCOL = "https"
@@ -89,6 +90,15 @@ class AmbariConfig:
# linux open-file limit
ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files'
+ # #### Command JSON file retention policies #####
+ # Keep all command-*.json files
+ COMMAND_FILE_RETENTION_POLICY_KEEP = 'keep'
+ # Remove command-*.json files if the operation was successful
+ COMMAND_FILE_RETENTION_POLICY_REMOVE_ON_SUCCESS = 'remove_on_success'
+ # Remove all command-*.json files when no longer needed
+ COMMAND_FILE_RETENTION_POLICY_REMOVE = 'remove'
+ # #### Command JSON file retention policies (end) #####
+
config = None
net = None
@@ -211,6 +221,45 @@ class AmbariConfig:
def host_scripts_dir(self):
return os.path.join(self.cache_dir, FileCache.HOST_SCRIPTS_CACHE_DIRECTORY)
+ @property
+ def command_file_retention_policy(self):
+ """
+ Returns the Agent's command file retention policy. This policy indicates what to do with the
+ command-*.json and status_command.json files after they are done being used to execute commands
+ from the Ambari server.
+
+ Possible policy values are:
+
+ * keep - Keep all command-*.json files
+ * remove - Remove all command-*.json files when no longer needed
+ * remove_on_success - Remove command-*.json files if the operation was successful
+
+ The policy value is expected to be set in the Ambari agent's ambari-agent.ini file, under the
+ [agent] section.
+
+ For example:
+ command_file_retention_policy=remove
+
+ However, if the value is not set, or set to an unexpected value, "keep" will be returned, since
+ this has been the (only) policy for past versions.
+
+ :rtype: string
+ :return: the command file retention policy, either "keep", "remove", or "remove_on_success"
+ """
+ policy = self.get('agent', self.COMMAND_FILE_RETENTION_POLICY_PROPERTY, default=self.COMMAND_FILE_RETENTION_POLICY_KEEP)
+ policies = [self.COMMAND_FILE_RETENTION_POLICY_KEEP,
+ self.COMMAND_FILE_RETENTION_POLICY_REMOVE,
+ self.COMMAND_FILE_RETENTION_POLICY_REMOVE_ON_SUCCESS]
+
+ if policy.lower() in policies:
+ return policy.lower()
+ else:
+ logger.warning('The configured command_file_retention_policy is invalid, returning "%s" instead: %s',
+ self.COMMAND_FILE_RETENTION_POLICY_KEEP,
+ policy)
+ return self.COMMAND_FILE_RETENTION_POLICY_KEEP
+
+
# TODO AMBARI-18733, change usages of this function to provide the home_dir.
@staticmethod
def getLogFile(home_dir=""):
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 48d7807..51a0d59 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -318,6 +318,12 @@ class CustomServiceOrchestrator():
"""
incremented_commands_for_component = False
+ # Make sure the return variable has been initialized
+ ret = None
+
+ # Make sure the json_path variable has been initialized
+ json_path = None
+
try:
command = self.generate_command(command_header)
script_type = command['commandParams']['script_type']
@@ -456,6 +462,8 @@ class CustomServiceOrchestrator():
if incremented_commands_for_component:
self.commands_for_component_in_progress[cluster_id][command['role']] -= 1
+ self.conditionally_remove_command_file(json_path, ret)
+
return ret
def command_canceled_reason(self, task_id):
@@ -672,3 +680,57 @@ class CustomServiceOrchestrator():
return resultList
+ def conditionally_remove_command_file(self, command_json_path, command_result):
+ """
+ Conditionally remove the specified command JSON file if it exists and if the configured
+ agent/command_file_retention_policy indicates to do so.
+
+ :param command_json_path: the absolute path to the command JSON file
+ :param command_result: the result structure containing the exit code for the command execution
+ :rtype: bool
+ :return: True, if the command JSON file was removed; False otherwise
+ """
+ removed_command_file = False
+
+ if os.path.exists(command_json_path):
+ command_file_retention_policy = self.config.command_file_retention_policy
+
+ if command_file_retention_policy == self.config.COMMAND_FILE_RETENTION_POLICY_REMOVE:
+ remove_command_file = True
+ logger.info(
+ 'Removing %s due to the command_file_retention_policy, %s',
+ command_json_path, command_file_retention_policy
+ )
+ elif command_file_retention_policy == self.config.COMMAND_FILE_RETENTION_POLICY_REMOVE_ON_SUCCESS:
+ if command_result and ('exitcode' in command_result):
+ exit_code = command_result['exitcode']
+ if exit_code == 0:
+ remove_command_file = True
+ logger.info(
+ 'Removing %s due to the command_file_retention_policy, %s, and exit code, %d',
+ command_json_path, command_file_retention_policy, exit_code
+ )
+ else:
+ remove_command_file = False
+ logger.info(
+ 'Not removing %s due to the command_file_retention_policy, %s, and exit code, %d',
+ command_json_path, command_file_retention_policy, exit_code
+ )
+ else:
+ remove_command_file = False
+ logger.info(
+ 'Not Removing %s due to the command_file_retention_policy, %s, and a missing exit code value',
+ command_json_path, command_file_retention_policy
+ )
+ else:
+ remove_command_file = False
+
+ if remove_command_file:
+ try:
+ os.remove(command_json_path)
+ removed_command_file = True
+ except Exception, e:
+ logger.error("Failed to remove %s due to error: %s", command_json_path, str(e))
+
+ return removed_command_file
+
diff --git a/ambari-agent/src/test/python/ambari_agent/TestAmbariConfig.py b/ambari-agent/src/test/python/ambari_agent/TestAmbariConfig.py
index c89644f..b958c3c 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestAmbariConfig.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestAmbariConfig.py
@@ -53,3 +53,32 @@ class TestAmbariConfig(TestCase):
open_files_ulimit = 128000
config.set_ulimit_open_files(open_files_ulimit)
self.assertEqual(config.get_ulimit_open_files(), open_files_ulimit)
+
+ def test_ambari_config_get_command_file_retention_policy(self):
+ config = AmbariConfig()
+
+ # unset value yields, "keep"
+ if config.has_option("agent", AmbariConfig.COMMAND_FILE_RETENTION_POLICY_PROPERTY):
+ config.remove_option("agent", AmbariConfig.COMMAND_FILE_RETENTION_POLICY_PROPERTY)
+ self.assertEqual(config.command_file_retention_policy,
+ AmbariConfig.COMMAND_FILE_RETENTION_POLICY_KEEP)
+
+ config.set("agent", AmbariConfig.COMMAND_FILE_RETENTION_POLICY_PROPERTY,
+ AmbariConfig.COMMAND_FILE_RETENTION_POLICY_KEEP)
+ self.assertEqual(config.command_file_retention_policy,
+ AmbariConfig.COMMAND_FILE_RETENTION_POLICY_KEEP)
+
+ config.set("agent", AmbariConfig.COMMAND_FILE_RETENTION_POLICY_PROPERTY,
+ AmbariConfig.COMMAND_FILE_RETENTION_POLICY_REMOVE)
+ self.assertEqual(config.command_file_retention_policy,
+ AmbariConfig.COMMAND_FILE_RETENTION_POLICY_REMOVE)
+
+ config.set("agent", AmbariConfig.COMMAND_FILE_RETENTION_POLICY_PROPERTY,
+ AmbariConfig.COMMAND_FILE_RETENTION_POLICY_REMOVE_ON_SUCCESS)
+ self.assertEqual(config.command_file_retention_policy,
+ AmbariConfig.COMMAND_FILE_RETENTION_POLICY_REMOVE_ON_SUCCESS)
+
+ # Invalid value yields, "keep"
+ config.set("agent", AmbariConfig.COMMAND_FILE_RETENTION_POLICY_PROPERTY, "invalid_value")
+ self.assertEqual(config.command_file_retention_policy,
+ AmbariConfig.COMMAND_FILE_RETENTION_POLICY_KEEP)