Commit 40765a17 authored by Jerome Mariette's avatar Jerome Mariette
Browse files

handle NFS latences

parent 7e1e8a72
...@@ -20,6 +20,7 @@ import sys ...@@ -20,6 +20,7 @@ import sys
import smtplib import smtplib
import socket import socket
import math import math
import shutil
try: try:
import DNS import DNS
...@@ -28,6 +29,27 @@ except: ...@@ -28,6 +29,27 @@ except:
DNS = None DNS = None
class ServerError(Exception): pass class ServerError(Exception): pass
def robust_rmtree(path, logger=None, max_retries=6):
"""Robustly tries to delete paths.
Retries several times (with increasing delays) if an OSError
occurs. If the final attempt fails, the Exception is propagated
to the caller.
"""
dt = 1
for i in range(max_retries):
try:
shutil.rmtree(path)
return
except OSError:
if logger:
logger.info('Unable to remove path: %s' % path)
logger.info('Retrying after %d seconds' % dt)
time.sleep(dt)
dt *= 2
# Final attempt, pass any Exceptions up to caller.
shutil.rmtree(path)
def display_error_message(msg): def display_error_message(msg):
sys.stderr.write("\033[91mError: "+msg+"\n\033[0m") sys.stderr.write("\033[91mError: "+msg+"\n\033[0m")
sys.exit(1) sys.exit(1)
......
...@@ -28,7 +28,6 @@ import threading ...@@ -28,7 +28,6 @@ import threading
import types import types
import logging import logging
import datetime import datetime
import shutil
from logging import FileHandler from logging import FileHandler
from ConfigParser import ConfigParser, NoOptionError from ConfigParser import ConfigParser, NoOptionError
...@@ -430,7 +429,7 @@ class Workflow(threading.Thread): ...@@ -430,7 +429,7 @@ class Workflow(threading.Thread):
def delete(self): def delete(self):
if self.get_status() in [self.STATUS_COMPLETED, self.STATUS_FAILED, self.STATUS_ABORTED]: if self.get_status() in [self.STATUS_COMPLETED, self.STATUS_FAILED, self.STATUS_ABORTED]:
shutil.rmtree(self.directory) utils.robust_rmtree(self.directory)
@staticmethod @staticmethod
def config_parser(arg_lines): def config_parser(arg_lines):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment