Commit 8d106b87 authored by Frédéric Escudié's avatar Frédéric Escudié
Browse files

Add dynamic outputs.

parent b8715a13
......@@ -26,14 +26,13 @@ from jflow.config_reader import JFlowConfigReader
from jflow.dataset import ArrayList
from jflow.utils import which
from jflow.parameter import *
from jflow.iotypes import DynamicOutput
from weaver.util import parse_string_list
class Component(object):
"""
"""
def __init__(self):
self.prefix = "default"
self.params_order = []
......@@ -41,8 +40,8 @@ class Component(object):
self.config_reader = JFlowConfigReader()
self.version = self.get_version()
self.batch_options = self.config_reader.get_component_batch_options(self.__class__.__name__)
def is_dynamic(self):
def is_dynamic(self):
return len(self.get_dynamic_outputs()) != 0
def get_dynamic_outputs(self):
......@@ -54,7 +53,7 @@ class Component(object):
if issubclass( attribute_value.__class__, DynamicOutput ):
dynamic_outputs.append( attribute_value )
return dynamic_outputs
def get_component_outputs(self, web_path):
outputs = {}
for attribute_value in self.__dict__.values():
......@@ -65,7 +64,7 @@ class Component(object):
elif issubclass( attribute_value.__class__, OutputFile):
outputs[os.path.basename(attribute_value)] = self._webify_workflow_outputs(web_path, attribute_value)
return outputs
def add_input_file(self, name, help, file_format="any", default=None, type="inputfile",
required=False, flag=None, group="default", display_name=None, add_to=None):
new_param = InputFile(name, help, flag=flag, file_format=file_format, default=default,
......@@ -78,7 +77,7 @@ class Component(object):
# otherwise, add it to the class itself
else:
new_param.component_nameid = self.get_nameid()
if default.__class__.__name__ == "OutputFileList" or default.__class__.__name__ == "OutputFile":
if issubclass( default.__class__, AbstractOutputFile ):
new_param.parent_component_nameid = default.component_nameid
self.params_order.append(name)
self.__setattr__(name, new_param)
......@@ -96,11 +95,11 @@ class Component(object):
# otherwise, add it to the class itself
else:
new_param.component_nameid = self.get_nameid()
if default.__class__.__name__ == "OutputFileList" or default.__class__.__name__ == "OutputFile":
if issubclass( default.__class__, AbstractOutputFile ):
new_param.parent_component_nameid = default.component_nameid
self.params_order.append(name)
self.__setattr__(name, new_param)
def add_parameter(self, name, help, default=None, type=types.StringType, choices=None,
required=False, flag=None, group="default", display_name=None, add_to=None):
new_param = Parameter(name, help, flag=flag, default=default, type=type, choices=choices,
......@@ -129,7 +128,7 @@ class Component(object):
else:
self.params_order.append(name)
self.__setattr__(name, new_param)
def add_output_file(self, name, help, file_format="any", filename=None, group="default", display_name=None, add_to=None):
filename = os.path.basename(filename)
new_param = OutputFile(name, help, default=os.path.join(self.output_directory, filename),
......@@ -144,7 +143,7 @@ class Component(object):
new_param.component_nameid = self.get_nameid()
self.params_order.append(name)
self.__setattr__(name, new_param)
def add_output_file_list(self, name, help, file_format="any", pattern='{basename_woext}.out',
items=None, group="default", display_name=None, add_to=None):
new_param = OutputFileList(name, help, default=self.get_outputs(pattern, items),
......@@ -159,12 +158,42 @@ class Component(object):
new_param.component_nameid = self.get_nameid()
self.params_order.append(name)
self.__setattr__(name, new_param)
def add_output_file_endswith(self, name, help, file_format="any", pattern='.stderr', behaviour="exclude",
group="default", display_name=None, add_to=None):
new_param = OutputFilesEndsWith(name, help, self.output_directory, pattern, include=(behaviour != "exclude"),
file_format="any", group="default", display_name=None)
# if this input should be added to a particular parameter
if add_to:
try:
self.__getattribute__(add_to).add_sub_parameter(new_param)
except: pass
# otherwise, add it to the class itself
else:
new_param.component_nameid = self.get_nameid()
self.params_order.append(name)
self.__setattr__(name, new_param)
def add_output_file_pattern(self, name, help, file_format="any", pattern='.stderr$', behaviour="exclude",
group="default", display_name=None, add_to=None):
new_param = OutputFilesPattern(name, help, self.output_directory, pattern, include=(behaviour != "exclude"),
file_format="any", group="default", display_name=None)
# if this input should be added to a particular parameter
if add_to:
try:
self.__getattribute__(add_to).add_sub_parameter(new_param)
except: pass
# otherwise, add it to the class itself
else:
new_param.component_nameid = self.get_nameid()
self.params_order.append(name)
self.__setattr__(name, new_param)
def _webify_workflow_outputs(self, web_path, path):
work_dir = self.config_reader.get_work_directory()
socket_opt = self.config_reader.get_socket_options()
return "http://" + socket_opt[0] + ":" + str(socket_opt[1]) + "/" + path.replace(work_dir, web_path)
def _longestCommonSubstr(self, data, clean_end=True):
substr = ''
if len(data) > 1 and len(data[0]) > 0:
......@@ -178,20 +207,20 @@ class Component(object):
while substr.endswith("_") or substr.endswith("-") or substr.endswith("."):
substr = substr[:-1]
return substr
def get_outputs(self, output_list=None, input_list=None):
"""
If `output_list` is a string template, then it may have the following
fields:
- `{fullpath}`, `{FULL}` -- Full input file path.
- `{basename}`, `{BASE}` -- Base input file name.
- `{fullpath_woext}`, `{FULLWE}` -- Full input file path without extension
- `{basename_woext}`, `{BASEWE}` -- Base input file name without extension
"""
"""
if output_list is None:
return []
if isinstance(output_list, str):
ilist = []
if not input_list or not '{' in str(output_list):
......
......@@ -15,6 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import re
import types
import datetime
import logging
......@@ -423,8 +424,13 @@ class InputFile(Parameter, AbstractIOFile):
Parameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices,
required=required, group=group, display_name=display_name)
class AbstractOutputFile(AbstractIOFile):
"""
@summary : Parent of all OutputFile(s) parameters.
"""
pass
class OutputFile(Parameter, AbstractIOFile):
class OutputFile(Parameter, AbstractOutputFile):
def __new__(self, name, help, file_format="any", default="", choices=None,
required=False, flag=None, group="default", display_name=None):
......@@ -472,17 +478,15 @@ class InputFileList(ParameterList, AbstractIOFile):
return list.__init__(self, [default])
elif default.__class__.__name__ == "InputFile":
return list.__init__(self, default)
elif default.__class__.__name__ == "OutputFile":
return list.__init__(self, default)
elif default.__class__.__name__ == "list":
return list.__init__(self, default)
elif default.__class__.__name__ == "InputFileList":
return list.__init__(self, default)
elif default.__class__.__name__ == "OutputFileList":
elif issubclass( default.__class__, AbstractOutputFile ):
return list.__init__(self, default)
class OutputFileList(ParameterList, AbstractIOFile):
class OutputFileList(ParameterList, AbstractOutputFile):
def __init__(self, name, help, file_format="any", default=None, choices=None,
required=False, flag=None, group="default", display_name=None):
if default == None: default = []
......@@ -493,3 +497,71 @@ class OutputFileList(ParameterList, AbstractIOFile):
return list.__init__(self, [default])
elif default.__class__.__name__ == "list":
return list.__init__(self, default)
class DynamicOutput(ParameterList, AbstractOutputFile):
"""
@warning : with this class of output, the component become dynamic.
"""
def update(self):
"""
This method is used at the end of component execution to update output list.
"""
raise NotImplementedError
class OutputFilesEndsWith(DynamicOutput):
def __init__(self, name, help, output_directory, end_str, include=True, file_format="any", choices=None,
required=False, flag=None, group="default", display_name=None):
"""
@warning : with this class of output, the component become dynamic.
@param output_directory : path to the directory where outputs will be created.
@param end_str : the end of the files names.
@param include : if true, the files with name terminated by end_str are added into output files.
If false, the files with name not terminated by end_str are added into output files.
"""
AbstractIOFile.__init__(self, file_format)
default = []
ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
required=required, group=group, display_name=display_name)
self.output_directory = output_directory
self.end_str = end_str
self.include = include
return list.__init__(self, default)
def update(self):
output_files = list()
for file in os.listdir( self.output_directory ):
if file.endswith( self.end_str ) and self.include :
output_files.append( os.path.join(self.output_directory, file) )
elif not file.endswith( self.end_str ) and not self.include:
output_files.append( os.path.join(self.output_directory, file) )
list.__init__(self, output_files)
class OutputFilesPattern(DynamicOutput):
def __init__(self, name, help, output_directory, pattern, include=True, file_format="any", choices=None,
required=False, flag=None, group="default", display_name=None):
"""
@warning : with this class of output, the component become dynamic.
@param output_directory : path to the directory where outputs will be created.
@param pattern : the pattern of (a part) the file names.
@param include : if true, the files with the pattern in file name are added into output files.
If false, the files with the pattern in file name are added into output files.
"""
AbstractIOFile.__init__(self, file_format)
default = []
ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
required=required, group=group, display_name=display_name)
self.output_directory = output_directory
self.pattern = pattern
self.include = include
return list.__init__(self, default)
def update(self):
output_files = list()
for file in os.listdir( self.output_directory ):
if self.include and re.search( self.pattern, file ) is not None:
output_files.append( os.path.join(self.output_directory, file) )
elif not self.include and re.search( self.pattern, file ) is None:
output_files.append( os.path.join(self.output_directory, file) )
return list.__init__(self, output_files)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment