Commit d2c6904b authored by Ibouniyamine Nabihoudine's avatar Ibouniyamine Nabihoudine
Browse files

automatic process function for components

parent 3a5cdb4e
......@@ -21,13 +21,18 @@ import inspect
import tempfile
import types
from operator import attrgetter
from jflow.workflows_manager import WorkflowsManager
from jflow.config_reader import JFlowConfigReader
from jflow.dataset import ArrayList
from jflow.utils import which, display_error_message
from jflow.parameter import *
from weaver.util import parse_string_list
from jflow.abstraction import MultiMap
from weaver.util import parse_string_list
from weaver.function import ShellFunction
from weaver.abstraction import Map
class Component(object):
......@@ -64,9 +69,11 @@ class Component(object):
return outputs
def add_input_file(self, name, help, file_format="any", default=None, type="inputfile",
required=False, flag=None, group="default", display_name=None, add_to=None):
required=False, flag=None, group="default", display_name=None, add_to=None,
cmd_format="", argpos=-1):
new_param = InputFile(name, help, flag=flag, file_format=file_format, default=default,
type=type, required=required, group=group, display_name=display_name)
type=type, required=required, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
# store where the parameter is coming from
new_param.linkTrace_nameid = self.get_nameid()
if issubclass( default.__class__, LinkTraceback ):
......@@ -86,7 +93,8 @@ class Component(object):
os.remove(os.path.join(self.output_directory, file))
def add_input_file_list(self, name, help, file_format="any", default=None, type="inputfile",
required=False, flag=None, group="default", display_name=None, add_to=None):
required=False, flag=None, group="default", display_name=None, add_to=None,
cmd_format="", argpos=-1):
if default == None:
inputs = []
elif issubclass(default.__class__, list):
......@@ -94,7 +102,8 @@ class Component(object):
else:
inputs = [IOFile(default, file_format, self.get_nameid(), None)]
new_param = InputFileList(name, help, flag=flag, file_format=file_format, default=inputs,
type=type, required=required, group=group, display_name=display_name)
type=type, required=required, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
# store where the parameter is coming from
new_param.linkTrace_nameid = self.get_nameid()
if issubclass( default.__class__, list ):
......@@ -119,9 +128,11 @@ class Component(object):
self.__setattr__(name, new_param)
def add_parameter(self, name, help, default=None, type=types.StringType, choices=None,
required=False, flag=None, group="default", display_name=None, add_to=None):
required=False, flag=None, group="default", display_name=None, add_to=None,
cmd_format="", argpos=-1):
new_param = ParameterFactory.factory(name, help, flag=flag, default=default, type=type, choices=choices,
required=required, group=group, display_name=display_name)
required=required, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
# if this input should be added to a particular parameter
if add_to:
try:
......@@ -133,10 +144,12 @@ class Component(object):
self.__setattr__(name, new_param)
def add_parameter_list(self, name, help, default=None, type=types.StringType, choices=None,
required=False, flag=None, group="default", display_name=None, add_to=None):
required=False, flag=None, group="default", display_name=None, add_to=None,
cmd_format="", argpos=-1):
if default == None: default = []
new_param = ParameterList(name, help, flag=flag, default=default, type=type, choices=choices,
required=required, group=group, display_name=display_name)
required=required, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
# if this input should be added to a particular parameter
if add_to:
try:
......@@ -147,10 +160,12 @@ class Component(object):
self.params_order.append(name)
self.__setattr__(name, new_param)
def add_output_file(self, name, help, file_format="any", filename=None, group="default", display_name=None, add_to=None):
def add_output_file(self, name, help, file_format="any", filename=None, group="default", display_name=None, add_to=None,
cmd_format="", argpos=-1):
filename = os.path.basename(filename)
new_param = OutputFile(name, help, default=os.path.join(self.output_directory, filename),
file_format=file_format, group=group, display_name=display_name)
file_format=file_format, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
# store where the parameter is coming from
new_param.linkTrace_nameid = self.get_nameid()
# if this input should be added to a particular parameter
......@@ -164,9 +179,11 @@ class Component(object):
self.__setattr__(name, new_param)
def add_output_file_list(self, name, help, file_format="any", pattern='{basename_woext}.out',
items=None, group="default", display_name=None, add_to=None):
items=None, group="default", display_name=None, add_to=None,
cmd_format="", argpos=-1):
default = [IOFile(file, file_format, self.get_nameid(), None) for file in self.get_outputs(pattern, items)]
new_param = OutputFileList(name, help, default=default, file_format=file_format, group=group, display_name=display_name)
new_param = OutputFileList(name, help, default=default, file_format=file_format, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
# store where the parameter is coming from
new_param.linkTrace_nameid = self.get_nameid()
# if this input should be added to a particular parameter
......@@ -180,9 +197,10 @@ class Component(object):
self.__setattr__(name, new_param)
def add_output_file_endswith(self, name, help, pattern, file_format="any", behaviour="include",
group="default", display_name=None, add_to=None):
group="default", display_name=None, add_to=None, cmd_format="", argpos=-1):
new_param = OutputFilesEndsWith(name, help, self.output_directory, pattern, include=(behaviour == "include"),
file_format=file_format, group=group, display_name=display_name)
file_format=file_format, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
# store where the parameter is coming from
new_param.linkTrace_nameid = self.get_nameid()
# if this input should be added to a particular parameter
......@@ -196,9 +214,10 @@ class Component(object):
self.__setattr__(name, new_param)
def add_output_file_pattern(self, name, help, pattern, file_format="any", behaviour="include",
group="default", display_name=None, add_to=None):
group="default", display_name=None, add_to=None, cmd_format="", argpos=-1):
new_param = OutputFilesPattern(name, help, self.output_directory, pattern, include=(behaviour == "exclude"),
file_format=file_format, group=group, display_name=display_name)
file_format=file_format, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
# store where the parameter is coming from
new_param.linkTrace_nameid = self.get_nameid()
# if this input should be added to a particular parameter
......@@ -210,7 +229,7 @@ class Component(object):
else:
self.params_order.append(name)
self.__setattr__(name, new_param)
def _longestCommonSubstr(self, data, clean_end=True):
substr = ''
if len(data) > 1 and len(data[0]) > 0:
......@@ -277,10 +296,99 @@ class Component(object):
def process(self):
"""
Run the component, has to be implemented by subclasses
Run the component, can be implemented by subclasses for a
more complex process
"""
# get all parameters
parameters = []
inputs = []
outputs = []
for param_name in self.params_order:
param = self.__getattribute__(param_name)
if isinstance(param, AbstractParameter) :
if isinstance(param, AbstractInputFile):
inputs.append(param)
elif isinstance(param, AbstractOutputFile):
outputs.append(param)
else :
parameters.append(param)
# sort parameters using argpos
parameters = sorted(parameters, key=attrgetter('argpos'))
inputs = sorted(inputs, key=attrgetter('argpos'))
outputs = sorted(outputs, key=attrgetter('argpos'))
filteredparams = []
commandline = self.get_command()
for p in parameters :
if isinstance(p, BoolParameter) :
if p:
commandline += " %s " % p.cmd_format
else :
commandline += " %s %s " % (p.cmd_format, p.default)
abstraction = self.get_abstraction()
if abstraction == None:
cpt = 1
for file in inputs + outputs :
if isinstance(file, InputFile) or isinstance(file, OutputFile):
commandline += ' %s $%s ' % (file.cmd_format, cpt)
cpt+=1
# input file list or output file list / pattern / ends with
else :
for e in file :
commandline += ' %s $%s ' % (file.cmd_format, cpt)
cpt+=1
function = ShellFunction( commandline, cmd_format='{EXE} {IN} {OUT}')
function(inputs=inputs, outputs=outputs)
# weaver map abstraction
elif abstraction == 'map' :
if not(len(inputs) == len(outputs) == 1) :
display_error_message("You can only have one type of input and one type of output for the map abstraction")
for file in inputs :
commandline += ' %s $1 ' % file.cmd_format
if isinstance(file, ParameterList) :
inputs = file
for file in outputs :
commandline += ' %s $2 ' % file.cmd_format
if isinstance(file, ParameterList) :
outputs = file
function = ShellFunction( commandline, cmd_format='{EXE} {IN} {OUT}')
exe = Map(function, inputs=inputs, outputs=outputs)
# jflow multimap
elif abstraction == 'multimap' :
cpt = 1
for file in inputs + outputs:
if not(isinstance(file, ParameterList)):
display_error_message("Multimap abstraction can be used only with ParameterList")
commandline += ' %s $%s ' % (file.cmd_format, cpt)
cpt+=1
function = ShellFunction( commandline, cmd_format='{EXE} {IN} {OUT}')
exe = MultiMap(function, inputs=inputs, outputs=outputs)
# anything other than that will be considered errored
else :
raise Exception('Unsupported abstraction %s ' % abstraction)
def get_command(self):
"""
get a path to an executable. Has to be implemented by subclasses
if the process has not been implemented
"""
raise NotImplementedError
def get_abstraction(self):
"""
get the abstraction. Has to be implemented by subclasses
if the process has not been implemented
"""
raise NotImplementedError
def get_version(self):
"""
Return the tool version, has to be implemented by subclasses
......
......@@ -314,7 +314,8 @@ class MiltipleAppendAction(argparse.Action):
class AbstractParameter(object):
def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
flag=None, action="store", sub_parameters=None, group="default", display_name=None):
flag=None, action="store", sub_parameters=None, group="default", display_name=None,
cmd_format="", argpos=-1):
self.name = name
self.help = help
......@@ -332,7 +333,9 @@ class AbstractParameter(object):
else: self.display_name = display_name
self.required = required
self.choices = choices
self.argpos = argpos
self.cmd_format = cmd_format
# Set parameter type
if type == "date":
self.type = date
......@@ -396,9 +399,9 @@ class IOFile(str, AbstractIOFile):
class MultiParameter(dict, AbstractParameter):
def __init__(self, name, help, required=False, flag=None, group="default", display_name=None):
def __init__(self, name, help, required=False, flag=None, group="default", display_name=None, cmd_format="", argpos=-1):
AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, group=group,
display_name=display_name)
display_name=display_name, cmd_format=cmd_format, argpos=argpos)
return dict.__init__(self, {})
def add_sub_parameter(self, param):
......@@ -431,9 +434,10 @@ class MultiParameter(dict, AbstractParameter):
class MultiParameterList(list, AbstractParameter):
def __init__(self, name, help, required=False, flag=None, group="default", display_name=None):
def __init__(self, name, help, required=False, flag=None, group="default", display_name=None, cmd_format="", argpos=-1):
AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag,
action="append", group=group, display_name=display_name)
action="append", group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
return list.__init__(self, [])
def add_sub_parameter(self, param):
......@@ -494,7 +498,7 @@ def none_decorator(fn):
class BoolParameter(int, AbstractParameter):
def __new__(self, name, help, default=False, type=types.BooleanType, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
bool_default = True
if default == None or default in ["False", "F", "false", "f", 0, "0"]:
bool_default = False
......@@ -508,9 +512,9 @@ class BoolParameter(int, AbstractParameter):
return val
def __init__(self, name, help, default=None, type=types.BooleanType, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
AbstractParameter.__init__(self, name, help, flag=flag, default=bool(default), type=type, choices=choices, required=required,
action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)
action="store", sub_parameters=sub_parameters, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
def __str__(self):
if self.is_None:
......@@ -541,7 +545,7 @@ class BoolParameter(int, AbstractParameter):
class IntParameter(int, AbstractParameter):
def __new__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
int_default = 0 if default == None else int(default)
val = int.__new__(self, int_default)
val.is_None = False if default != None else True
......@@ -553,9 +557,9 @@ class IntParameter(int, AbstractParameter):
return val
def __init__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
AbstractParameter.__init__( self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
action="store", sub_parameters=sub_parameters, group=group, display_name=display_name )
action="store", sub_parameters=sub_parameters, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
def __str__(self):
if self.is_None:
......@@ -586,7 +590,7 @@ class IntParameter(int, AbstractParameter):
class FloatParameter(float, AbstractParameter):
def __new__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
float_default = 0.0 if default == None else float(default)
val = float.__new__(self, float_default)
val.is_None = False if default != None else True
......@@ -598,9 +602,9 @@ class FloatParameter(float, AbstractParameter):
return val
def __init__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)
action="store", sub_parameters=sub_parameters, group=group, display_name=display_name,cmd_format=cmd_format, argpos=argpos )
def __str__(self):
if self.is_None:
......@@ -631,7 +635,7 @@ class FloatParameter(float, AbstractParameter):
class StrParameter(str, AbstractParameter):
def __new__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
str_default = "" if default == None else str(default)
val = str.__new__(self, str_default)
val.is_None = False if default != None else True
......@@ -643,9 +647,9 @@ class StrParameter(str, AbstractParameter):
return val
def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)
action="store", sub_parameters=sub_parameters, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
def __str__(self):
if self.is_None:
......@@ -678,7 +682,7 @@ class StrParameter(str, AbstractParameter):
class DateParameter(datetime.datetime, AbstractParameter):
def __new__(self, name, help, default=None, type=date, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
date_default = datetime.datetime.today()
if default != None and issubclass(default.__class__, datetime.datetime):
date_default = default
......@@ -694,12 +698,12 @@ class DateParameter(datetime.datetime, AbstractParameter):
return val
def __init__(self, name, help, default=None, type=date, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
if default != None and not issubclass(default.__class__, datetime.datetime):
date_default = date(default)
default = datetime.datetime(date_default.year, date_default.month, date_default.day)
AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)
action="store", sub_parameters=sub_parameters, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
def __str__(self):
if self.is_None:
......@@ -794,7 +798,7 @@ class AbstractOutputFile(AbstractIOFile):
class InputFile(StrParameter, AbstractInputFile):
def __new__(self, name, help, file_format="any", default="", type="localfile", choices=None,
required=False, flag=None, group="default", display_name=None, size_limit="0"):
required=False, flag=None, group="default", display_name=None, size_limit="0", cmd_format="", argpos=-1):
if hasattr(type, '__call__'):
type2test = type.__name__
else: type2test = type
......@@ -804,13 +808,13 @@ class InputFile(StrParameter, AbstractInputFile):
+ "', '".join(INPUTFILE_TYPES)+"'")
return StrParameter.__new__(self, name, help, flag=flag, default=default, type=type, choices=choices,
required=required, group=group, display_name=display_name)
required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None,
required=False, flag=None, group="default", display_name=None, size_limit="0"):
required=False, flag=None, group="default", display_name=None, size_limit="0", cmd_format="", argpos=-1):
AbstractInputFile.__init__(self, file_format, size_limit)
StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices,
required=required, group=group, display_name=display_name)
required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
def get_type(self):
return self.type.__name__+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit
......@@ -839,24 +843,28 @@ class InputFile(StrParameter, AbstractInputFile):
class OutputFile(StrParameter, AbstractOutputFile):
def __new__(self, name, help, file_format="any", default="", choices=None,
required=False, flag=None, group="default", display_name=None):
required=False, flag=None, group="default", display_name=None,
cmd_format="", argpos=-1):
return StrParameter.__new__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
required=required, group=group, display_name=display_name)
required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
def __init__(self, name, help, file_format="any", default="", choices=None,
required=False, flag=None, group="default", display_name=None):
required=False, flag=None, group="default", display_name=None,
cmd_format="", argpos=-1):
AbstractIOFile.__init__(self, file_format)
StrParameter.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
required=required, group=group, display_name=display_name)
required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
class ParameterList(list, AbstractParameter):
def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
flag=None, sub_parameters=None, group="default", display_name=None,
cmd_format="", argpos=-1):
if default == None: default = []
AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
action="append", sub_parameters=sub_parameters, group=group, display_name=display_name)
action="append", sub_parameters=sub_parameters, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
if default.__class__.__name__ == "str":
return list.__init__(self, [default])
elif default.__class__.__name__ == "list":
......@@ -872,7 +880,8 @@ class ParameterList(list, AbstractParameter):
class InputFileList(ParameterList, AbstractInputFile):
def __init__(self, name, help, file_format="any", default=None, type="localfile", choices=None,
required=False, flag=None, group="default", display_name=None, size_limit="0"):
required=False, flag=None, group="default", display_name=None, size_limit="0",
cmd_format="", argpos=-1):
if default == None: default = []
if hasattr(type, '__call__'):
......@@ -889,7 +898,8 @@ class InputFileList(ParameterList, AbstractInputFile):
AbstractInputFile.__init__(self, file_format, size_limit)
ParameterList.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices,
required=required, group=group, display_name=display_name)
required=required, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
if default.__class__.__name__ == "str":
return list.__init__(self, [default])
......@@ -944,11 +954,13 @@ class InputFileList(ParameterList, AbstractInputFile):
class OutputFileList(ParameterList, AbstractOutputFile):
def __init__(self, name, help, file_format="any", default=None, choices=None,
required=False, flag=None, group="default", display_name=None):
required=False, flag=None, group="default", display_name=None,
cmd_format="", argpos=-1):
if default == None: default = []
AbstractIOFile.__init__(self, file_format)
ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
required=required, group=group, display_name=display_name)
required=required, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
if default.__class__.__name__ == "str":
return list.__init__(self, [default])
elif default.__class__.__name__ == "list":
......@@ -969,7 +981,7 @@ class DynamicOutput(ParameterList, AbstractOutputFile):
class OutputFilesEndsWith(DynamicOutput):
def __init__(self, name, help, output_directory, end_str, include=True, file_format="any", choices=None,
required=False, flag=None, group="default", display_name=None):
required=False, flag=None, group="default", display_name=None, cmd_format="", argpos=-1):
"""
@warning : with this class of output, the component become dynamic.
@param output_directory : path to the directory where outputs will be created.
......@@ -980,7 +992,8 @@ class OutputFilesEndsWith(DynamicOutput):
AbstractIOFile.__init__(self, file_format)
default = []
ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
required=required, group=group, display_name=display_name)
required=required, group=group, display_name=display_name, cmd_format=cmd_format,
argpos=argpos)
self.output_directory = output_directory
self.end_str = end_str
self.include = include
......@@ -999,7 +1012,7 @@ class OutputFilesEndsWith(DynamicOutput):
class OutputFilesPattern(DynamicOutput):
def __init__(self, name, help, output_directory, pattern, include=True, file_format="any", choices=None,
required=False, flag=None, group="default", display_name=None):
required=False, flag=None, group="default", display_name=None, cmd_format="", argpos=-1):
"""
@warning : with this class of output, the component become dynamic.
@param output_directory : path to the directory where outputs will be created.
......@@ -1010,7 +1023,8 @@ class OutputFilesPattern(DynamicOutput):
AbstractIOFile.__init__(self, file_format)
default = []
ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
required=required, group=group, display_name=display_name)
required=required, group=group, display_name=display_name, cmd_format=cmd_format,
argpos=argpos)
self.output_directory = output_directory
self.pattern = pattern
self.include = include
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment