handle NFS latences

src/jflow/utils.py

@@ -20,6 +20,7 @@ import sys
 import smtplib
 import socket
 import math
+import shutil
 
 try:
     import DNS
@@ -28,6 +29,27 @@ except:
     DNS = None
     class ServerError(Exception): pass
 
+def robust_rmtree(path, logger=None, max_retries=6):
+    """Robustly tries to delete paths.
+    Retries several times (with increasing delays) if an OSError
+    occurs.  If the final attempt fails, the Exception is propagated
+    to the caller.
+    """
+    dt = 1
+    for i in range(max_retries):
+        try:
+            shutil.rmtree(path)
+            return
+        except OSError:
+            if logger:
+                logger.info('Unable to remove path: %s' % path)
+                logger.info('Retrying after %d seconds' % dt)
+            time.sleep(dt)
+            dt *= 2
+
+    # Final attempt, pass any Exceptions up to caller.
+    shutil.rmtree(path)
+
 def display_error_message(msg):
     sys.stderr.write("\033[91mError: "+msg+"\n\033[0m")
     sys.exit(1)

src/jflow/workflow.py

@@ -28,7 +28,6 @@ import threading
 import types
 import logging
 import datetime
-import shutil
 
 from logging import FileHandler
 from ConfigParser import ConfigParser, NoOptionError
@@ -430,7 +429,7 @@ class Workflow(threading.Thread):
 
     def delete(self):
         if self.get_status() in [self.STATUS_COMPLETED, self.STATUS_FAILED, self.STATUS_ABORTED]:
-            shutil.rmtree(self.directory)
+            utils.robust_rmtree(self.directory)
 
     @staticmethod
     def config_parser(arg_lines):