add DynamicOutputs to handle dynamc components

src/jflow/component.py

@@ -24,6 +24,7 @@ from jflow.workflows_manager import WorkflowsManager
 from jflow.config_reader import JFlowConfigReader
 from jflow.dataset import ArrayList
 from jflow.utils import which
+from jflow.iotypes import DynamicOutput
 
 from weaver.util import parse_string_list
 
@@ -32,16 +33,25 @@ class Component(object):
     """
     """
         
-    def __init__(self, is_dynamic=False):
-        """
-          @param is_dynamic: if true, this component has dynamic inputs/outputs
-        """
-        self.is_dynamic = is_dynamic
+    def __init__(self):
         self.prefix = "default"
         self.output_directory = None
         self.config_reader = JFlowConfigReader()
         self.version = self.get_version()
         self.batch_options = self.config_reader.get_component_batch_options(self.__class__.__name__)
+    
+    def is_dynamic(self):           
+        return len(self.get_dynamic_outputs()) != 0
+
+    def get_dynamic_outputs(self):
+        """
+         @return : the list of outputs updated at the end of component execution.
+        """
+        dynamic_outputs = list()
+        for attribute_value in self.__dict__.values():
+            if issubclass( attribute_value.__class__, DynamicOutput ):
+                dynamic_outputs.append( attribute_value )
+        return dynamic_outputs
         
     def _longestCommonSubstr(self, data, clean_end=True):
         substr = ''

src/jflow/iotypes.py

@@ -15,7 +15,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import os
+import os, re
+import abc
+from _pyio import __metaclass__
+
 
 class Formats(object):
     ANY = "any"
@@ -23,6 +26,9 @@ class Formats(object):
     FASTQ = "fastq"
     FASTA = "fasta"
     SFF = "sff"
+    QUAL = "qual"
+    FLOW = "flow"
+    HTML = "html"
     
 # 
 # Inputs classes
@@ -55,6 +61,10 @@ class InputFileList(list):
             return list.__init__(self, inputs)
         elif inputs.__class__.__name__ == "OutputFileList" and (inputs.format == self.format or self.format == Formats.ANY):
             return list.__init__(self, inputs)
+        elif inputs.__class__.__name__ == "OutputFileEndsWith" and (inputs.format == self.format or self.format == Formats.ANY):
+            return list.__init__(self, inputs)
+        elif inputs.__class__.__name__ == "OutputFilePattern" and (inputs.format == self.format or self.format == Formats.ANY):
+            return list.__init__(self, inputs)
         elif (inputs.__class__.__name__ == "OutputFile" or inputs.__class__.__name__ == "OutputFileList") and \
             not (inputs.format == self.format or self.format == Formats.ANY):
             raise IOError(inputs.format+" format is incompatible with allowed format "+self.format)
@@ -94,6 +104,19 @@ class InputDirectoryList(list):
 # 
 # Outputs classes
 #
+class DynamicOutput(list):
+    """        
+     @warning : with this class of output, the component become dynamic.
+    """
+    __metaclass__ = abc.ABCMeta
+    
+    @abc.abstractmethod
+    def update(self):
+        """ 
+         This method is used at the end of component execution to update output list.
+        """
+        raise NotImplementedError
+    
 class OutputFile(str):
     
     def __new__(self, output, format=Formats.ANY):
@@ -105,6 +128,78 @@ class OutputFile(str):
     
     def check(self):
         return "TODO: check output file"
+        
+class OutputFileList(list):
+    
+    def __init__(self, outputs, format=Formats.ANY):
+        self.format = format
+        if outputs.__class__.__name__ == "str":
+            return list.__init__(self, [outputs])
+        elif outputs.__class__.__name__ == "list":
+            return list.__init__(self, outputs)
+        else:
+            raise IOError(outputs.__class__.__name__+" cannot be used as OutputFileList")
+
+    def check(self):
+        return "TODO: check output file list"
+
+class OutputFilePattern(DynamicOutput):
+    
+    def __init__(self, output_directory, pattern, format=Formats.ANY, include=True):
+        """        
+         @warning : with this class of output, the component become dynamic.
+         @param output_directory : path to the directory where outputs will be created.
+         @param pattern : the pattern of (a part) the file names.
+         @param format : the files format.
+         @param include : if true, the files with the pattern in file name are added into output files.
+                          If false, the files with the pattern in file name are added into output files.
+        """
+        self.output_directory = output_directory
+        self.pattern = pattern
+        self.format = format
+        self.include = include
+        return list.__init__(self, [])
+
+    def update(self):
+        output_files = list()
+        for file in os.listdir( self.output_directory ):
+            if self.include and re.search( self.pattern, file ) is not None:
+                output_files.append( os.path.join(self.output_directory, file) )
+            elif not self.include and re.search( self.pattern, file ) is None:
+                output_files.append( os.path.join(self.output_directory, file) )
+        return list.__init__(self, output_files)
+    
+    def check(self):
+        return "TODO: check output file list"
+
+class OutputFileEndsWith(DynamicOutput):
+    
+    def __init__(self, output_directory, end_str, format=Formats.ANY, include=True):
+        """        
+         @warning : with this class of output, the component become dynamic.
+         @param output_directory : path to the directory where outputs will be created.
+         @param end_str : the end of the files names.
+         @param format : the files format.
+         @param include : if true, the files with name terminated by end_str are added into output files.
+                          If false, the files with name not terminated by end_str are added into output files.
+        """
+        self.output_directory = output_directory
+        self.end_str = end_str
+        self.format = format
+        self.include = include
+        return list.__init__(self, [])
+
+    def update(self):
+        output_files = list()
+        for file in os.listdir( self.output_directory ):
+            if file.endswith( self.end_str ) and self.include :
+                output_files.append( os.path.join(self.output_directory, file) )
+            elif not file.endswith( self.end_str ) and not self.include:
+                output_files.append( os.path.join(self.output_directory, file) )
+        return list.__init__(self, output_files)
+    
+    def check(self):
+        return "TODO: check output file list"
 
 class OutputFileList(list):
     

src/jflow/workflow.py

@@ -209,8 +209,11 @@ class Workflow(threading.Thread):
             # add the component
             self.components.append(cmpt_object)
             # if this one require a dynamic pipeline, execute the first part
-            if cmpt_object.is_dynamic:
+            if cmpt_object.is_dynamic():
                 self._execute_weaver()
+                # update outputs
+                for output in cmpt_object.get_dynamic_outputs():
+                    output.update()
             return cmpt_object
         else:
             raise ImportError(component_name + " component cannot be loaded, available components are: {0}".format(", ".join(self.comp_pckg.keys())))