improve the way workflows info are gathered ...

src/jflow/workflow.py

@@ -53,7 +53,7 @@ class Workflow(threading.Thread):
 
     PROPERTIES_FILE_NAME = "workflow.properties"
     MAKEFLOW_LOG_FILE_NAME = "Makeflow.makeflowlog"
-    DUMP_FILE_NAME = "workflow.dump"
+    DUMP_FILE_NAME = ".workflow.dump"
     WORKING = ".working"
     STATUS_STARTED = "started"
     STATUS_COMPLETED = "completed"

src/jflow/workflows_manager.py

@@ -33,9 +33,11 @@ class WorkflowsManager(object):
     """
     
     IDS_FILE_NAME = "jflowIDs.txt"
+    WORKFLOWS_DUMP_FILE_NAME = ".workflows.dump"
     WF_DIRECTORY_PREFIX = "wf"
     
     lock_ids_file = threading.Lock()
+    lock_workflows_file = threading.Lock()
     
     def __init__(self):
         """ 
@@ -43,8 +45,20 @@ class WorkflowsManager(object):
         self.config_reader = JFlowConfigReader()
         if not os.path.isdir(self.get_output_directory()):
             os.makedirs(self.get_output_directory(), 0751)
+        self.dump_file = os.path.join(self.get_output_directory(), self.WORKFLOWS_DUMP_FILE_NAME)
+        # if the file already exists, load all the workflows dump path
+        if os.path.isfile(self.dump_file):
+            self.workflows_dump = pickle.load(open(self.dump_file))
+        else:
+            self.workflows_dump = {}
         self.ids_file = os.path.join(self.get_output_directory(), self.IDS_FILE_NAME)
     
+    def _add_workflow(self, workflow):
+        WorkflowsManager.lock_workflows_file.acquire()
+        self.workflows_dump[utils.get_nb_string(workflow.id)] = workflow.dump_path
+        pickle.dump(self.workflows_dump, open(self.dump_file, "w"))
+        WorkflowsManager.lock_workflows_file.release()
+    
     def get_available_workflows(self, parameters_section="parameters"):
         wf_instances = []
         # Load all modules within the workflow module
@@ -70,26 +84,21 @@ class WorkflowsManager(object):
             for class_name, obj in inspect.getmembers(sys.modules[modname], inspect.isclass):
                 if class_name == workflow_class: workflow = obj(args, self.get_next_id(), function, parameters_section)
         workflow.start()
+        # Add the workflow dump path to the workflows dump
+        self._add_workflow(workflow)
         return workflow
         
     def get_output_directory(self):
         return self.config_reader.get_work_directory()
     
     def get_workflows(self):
-        from jflow.workflow import Workflow
         workflows = []
-        for wf_name in os.listdir(self.get_output_directory()):
-            wf_name_path = os.path.join(self.get_output_directory(), wf_name)
-            # If it's a workflow directory
-            if os.path.isdir(wf_name_path):
-                for wf in os.listdir(wf_name_path):
-                    wf_path = os.path.join(wf_name_path, wf)
-                    workflow_dump_path = os.path.join(wf_path, Workflow.DUMP_FILE_NAME)
-                    try:
-                        workflow_dump = open(workflow_dump_path, "rb")
-                        workflows.append(pickle.load(workflow_dump))
-                        workflow_dump.close()
-                    except: pass
+        for workflow_id in self.workflows_dump:
+            try:
+                workflow_dump = open(self.workflows_dump[workflow_id], "rb")
+                workflows.append(pickle.load(workflow_dump))
+                workflow_dump.close()
+            except: pass
         return workflows
 
     def get_workflow_by_class(self, workflow_class):
@@ -103,25 +112,13 @@ class WorkflowsManager(object):
         return None
     
     def get_workflow(self, workflow_id):
-        from jflow.workflow import Workflow
-        search_directory = self.WF_DIRECTORY_PREFIX + utils.get_nb_string(workflow_id)
-        workflow_dump_path = None
-        for wf_name in os.listdir(self.get_output_directory()):
-            wf_name_path = os.path.join(self.get_output_directory(), wf_name)
-            # If it's a workflow directory
-            if os.path.isdir(wf_name_path):
-                for wf in os.listdir(wf_name_path):
-                    wf_path = os.path.join(wf_name_path, wf)
-                    if wf == search_directory:
-                        if os.path.isfile(os.path.join(wf_path, Workflow.DUMP_FILE_NAME)):
-                            workflow_dump_path = os.path.join(wf_path, Workflow.DUMP_FILE_NAME)
-                        break
-        if workflow_dump_path:
-            workflow_dump = open(workflow_dump_path, "rb")
+        rworkflow_id = utils.get_nb_string(workflow_id)
+        if self.workflows_dump.has_key(rworkflow_id):
+            workflow_dump = open(self.workflows_dump[rworkflow_id], "rb")
             workflow = pickle.load(workflow_dump)
             workflow_dump.close()
         else:
-            raise ValueError("Workflow with id " + str(workflow_id) + " cannot be retrived!")
+            raise ValueError("Workflow with id " + str(rworkflow_id) + " cannot be retrived!")
         return workflow
 
     def get_workflow_directory(self, wname, wid):