[ambari] 01/02: AMBARI-23184. Metrics Collector Install failed on HA cluster (aonishuk)

[ao...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git

commit 60fc46a0afb3b4042e65f1b93ec9eb9d131aafff
Author: Andrew Onishuk <ao...@hortonworks.com>
AuthorDate: Fri Mar 9 12:26:32 2018 +0200

    AMBARI-23184. Metrics Collector Install failed on HA cluster (aonishuk)
---
 ambari-agent/src/main/python/ambari_agent/FileCache.py | 12 +++++++++++-
 ambari-agent/src/main/python/ambari_agent/Utils.py     | 12 +++++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/ambari-agent/src/main/python/ambari_agent/FileCache.py b/ambari-agent/src/main/python/ambari_agent/FileCache.py
index 6994044..1df4926 100644
--- a/ambari-agent/src/main/python/ambari_agent/FileCache.py
+++ b/ambari-agent/src/main/python/ambari_agent/FileCache.py
@@ -26,6 +26,8 @@ import zipfile
 import urllib2
 import urllib
 
+from ambari_agent.Utils import execute_with_retries
+
 logger = logging.getLogger()
 
 class CachingException(Exception):
@@ -257,13 +259,21 @@ class FileCache():
     directory and any parent directories if needed. May throw exceptions
     on permission problems
     """
+    CLEAN_DIRECTORY_TRIES = 5
+    CLEAN_DIRECTORY_TRY_SLEEP = 0.25
+
     logger.debug("Invalidating directory {0}".format(directory))
     try:
       if os.path.exists(directory):
         if os.path.isfile(directory): # It would be a strange situation
           os.unlink(directory)
         elif os.path.isdir(directory):
-          shutil.rmtree(directory)
+          """
+          Execute shutil.rmtree(directory) multiple times.
+          Reason: race condition, where a file (e.g. *.pyc) in deleted directory
+          is created during function is running, causing it to fail.
+          """
+          execute_with_retries(CLEAN_DIRECTORY_TRIES, CLEAN_DIRECTORY_TRY_SLEEP, OSError, shutil.rmtree, directory)
         # create directory itself and any parent directories
       os.makedirs(directory)
     except Exception, err:
diff --git a/ambari-agent/src/main/python/ambari_agent/Utils.py b/ambari-agent/src/main/python/ambari_agent/Utils.py
index 370597e..f100f14 100644
--- a/ambari-agent/src/main/python/ambari_agent/Utils.py
+++ b/ambari-agent/src/main/python/ambari_agent/Utils.py
@@ -18,6 +18,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 import os
+import time
 import threading
 import collections
 from functools import wraps
@@ -214,4 +215,13 @@ def lazy_property(undecorated):
       setattr(self, name, v)
       return v
 
-  return decorated
\ No newline at end of file
+  return decorated
+
+def execute_with_retries(tries, try_sleep, rety_exception_class, func, *args, **kwargs):
+  for i in range(tries):
+    try:
+      func(*args, **kwargs)
+    except rety_exception_class:
+      if i==tries-1:
+        raise
+      time.sleep(try_sleep)
\ No newline at end of file

-- 
To stop receiving notification emails like this one, please contact
aonishuk@apache.org.