Commit 011f3a9e authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Accept non-iterable parameters for map and multimap abstractions

parent 548ea197
......@@ -17,7 +17,7 @@
[global]
# uncomment and set if not in the PATH, should be version >= 4.4.3
#makeflow = /usr/bin/makeflow
makeflow = /home/fcabanettes/cctools6/bin/makeflow
# batch system type: local, condor, sge, moab, cluster, wq, hadoop, mpi-queue
batch_system_type = local
# add these options to all batch submit files
......@@ -47,19 +47,20 @@ from_password =
#message =
[storage]
# In this section, ###USER### (if it's used) is replaced by $USER environment variable.
# where should be written the log file
log_file = <path>/jflow.log
log_file = /home/fcabanettes/jflow-work/jflow.log
# Where should the pipelines write results, should be accessible
# by all cluster nodes
work_directory = <path>/work
work_directory = /home/fcabanettes/jflow-work/work
# Where should the pipelines write temporary files, should be
# accessible by all cluster nodes
tmp_directory = <path>/tmp
tmp_directory = /home/fcabanettes/jflow-work/tmp
[softwares]
# uncomment and set if not in the PATH
#bwa = /usr/bin/bwa
bwa = /usr/bin/bwa
cutadapt = /usr/local/bin/cutadapt
STAR = /usr/bin/STAR
#samtools = /usr/bin/samtools
# Set cluster parameters of some components
......
......@@ -27,7 +27,66 @@ from weaver.options import Options
from weaver.abstraction import Abstraction
class MultiMap(Abstraction):
class AbstractionWargs(Abstraction):
""" The base Abstraction class.
**Positional Arguments**:
- `function` -- Function to apply (Function, string, string format)
**Keyword Arguments**:
- `inputs` -- Inputs to function
- `outputs` -- Output of function
- `includes` -- Files to include for each task.
- `native` -- Whether or not to use native abstraction if available.
- `group` -- Number of tasks to inline.
- `collect` -- Whether or not to mark files for garbage collection.
- `local` -- Whether or not to force local execution.
`inputs` and `includes` are parsed using
:func:`~weaver.data.parse_input_list` and must be in a form acceptable tot
hat function. Likewise, `outputs` is parsed by
:func:`~weaver.data.parse_output_list` and `function` is parsed by
:func:`~weaver.function.parse_function`.
"""
Counter = None
def __init__(self, function, inputs=None, outputs=None, includes=None,
native=False, group=None, collect=False, local=False, arguments=None):
Abstraction.__init__(self, function, inputs, outputs, includes, native, group, collect, local)
self.arguments = arguments
class Map(AbstractionWargs):
""" Weaver Map Abstraction.
This Abstraction enables the following pattern of execution:
Map(f, inputs, outputs)
In this case, the :class:`Function` *f* is applied to each item in
*inputs* to generate the corresponding *outputs*.
"""
Counter = itertools.count()
@cache_generation
def _generate(self):
with self:
debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self))
function = parse_function(self.function)
inputs = parse_input_list(self.inputs)
outputs = parse_output_list(self.outputs, inputs)
includes = parse_input_list(self.includes)
for i, o in zip(inputs, outputs):
with Options(local=self.options.local, collect=[i] if self.collect else None):
yield function(i, o, self.arguments, includes)
class MultiMap(AbstractionWargs):
""" Weaver MultiMap Abstraction.
This Abstraction enables the following pattern of execution:
......@@ -55,6 +114,7 @@ class MultiMap(Abstraction):
@cache_generation
def _generate(self):
print("ARGS: " + str(self.arguments))
with self:
debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self))
......@@ -110,5 +170,5 @@ class MultiMap(Abstraction):
iteration_outputs = parse_output_list(self.outputs, input_pattern)
with Options(local=self.options.local):
yield function(iteration_inputs, iteration_outputs, None, includes)
yield function(iteration_inputs, iteration_outputs, self.arguments, includes)
......@@ -33,7 +33,7 @@ from jflow.abstraction import MultiMap
from weaver.util import parse_string_list
from weaver.function import ShellFunction
from weaver.abstraction import Map
from jflow.abstraction import Map
from weaver.function import PythonFunction
......@@ -554,9 +554,6 @@ class Component(object):
add_path=None, collect=False, local=False, map=False, cmd_format=""):
if map:
if arguments != [] :
logging.getLogger("jflow").exception("add_python_execution: '" + function.__name__ + "' arguments parameter not allowed with map!")
raise Exception("add_python_execution: '" + function.__name__ + "' arguments parameter not allowed with map!" )
if not issubclass(inputs.__class__, list) or not issubclass(outputs.__class__, list):
logging.getLogger("jflow").exception("add_python_execution: '" + function.__name__ + "' map requires a list as inputs and output!")
raise Exception("add_python_execution: '" + function.__name__ + "' map requires a list as inputs and output!")
......@@ -577,7 +574,8 @@ class Component(object):
if not isinstance(includes, list):
includes=[includes]
if map:
MultiMap(py_function, inputs=new_inputs, outputs=new_outputs, includes=includes+includes_in, collect=collect, local=local)
MultiMap(py_function, inputs=new_inputs, outputs=new_outputs, includes=includes+includes_in,
collect=collect, local=local, arguments=arguments)
else:
py_function(inputs=new_inputs, outputs=new_outputs, arguments=arguments, includes=includes+includes_in)
......@@ -591,11 +589,7 @@ class Component(object):
if map :
# if input and output are list or filelist
if issubclass(inputs.__class__, list) and issubclass(outputs.__class__, list) :
# arguments cannot be set with
if arguments != [] :
logging.getLogger("jflow").exception("add_shell_execution: '" + source + "' arguments parameter not allowed with map")
raise Exception("add_shell_execution: '" + source + "' arguments parameter not allowed with map" )
MultiMap(shell_function,inputs=inputs, outputs=outputs, includes=includes, collect=collect, local=local)
MultiMap(shell_function,inputs=inputs, outputs=outputs, includes=includes, collect=collect, local=local, arguments=arguments)
else :
logging.getLogger("jflow").exception("add_shell_execution: '" + source + "' map requires a list as inputs and output")
raise Exception("add_shell_execution: '" + source + "' map requires a list as inputs and output")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment