Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
genotoul-bioinfo
jflow
Commits
a8d1eca7
Commit
a8d1eca7
authored
Jan 30, 2014
by
Jerome Mariette
Browse files
add DynamicOutputs to handle dynamc components
parent
5ef8d333
Changes
3
Hide whitespace changes
Inline
Side-by-side
src/jflow/component.py
View file @
a8d1eca7
...
...
@@ -24,6 +24,7 @@ from jflow.workflows_manager import WorkflowsManager
from
jflow.config_reader
import
JFlowConfigReader
from
jflow.dataset
import
ArrayList
from
jflow.utils
import
which
from
jflow.iotypes
import
DynamicOutput
from
weaver.util
import
parse_string_list
...
...
@@ -32,16 +33,25 @@ class Component(object):
"""
"""
def
__init__
(
self
,
is_dynamic
=
False
):
"""
@param is_dynamic: if true, this component has dynamic inputs/outputs
"""
self
.
is_dynamic
=
is_dynamic
def
__init__
(
self
):
self
.
prefix
=
"default"
self
.
output_directory
=
None
self
.
config_reader
=
JFlowConfigReader
()
self
.
version
=
self
.
get_version
()
self
.
batch_options
=
self
.
config_reader
.
get_component_batch_options
(
self
.
__class__
.
__name__
)
def
is_dynamic
(
self
):
return
len
(
self
.
get_dynamic_outputs
())
!=
0
def
get_dynamic_outputs
(
self
):
"""
@return : the list of outputs updated at the end of component execution.
"""
dynamic_outputs
=
list
()
for
attribute_value
in
self
.
__dict__
.
values
():
if
issubclass
(
attribute_value
.
__class__
,
DynamicOutput
):
dynamic_outputs
.
append
(
attribute_value
)
return
dynamic_outputs
def
_longestCommonSubstr
(
self
,
data
,
clean_end
=
True
):
substr
=
''
...
...
src/jflow/iotypes.py
View file @
a8d1eca7
...
...
@@ -15,7 +15,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import
os
import
os
,
re
import
abc
from
_pyio
import
__metaclass__
class
Formats
(
object
):
ANY
=
"any"
...
...
@@ -23,6 +26,9 @@ class Formats(object):
FASTQ
=
"fastq"
FASTA
=
"fasta"
SFF
=
"sff"
QUAL
=
"qual"
FLOW
=
"flow"
HTML
=
"html"
#
# Inputs classes
...
...
@@ -55,6 +61,10 @@ class InputFileList(list):
return
list
.
__init__
(
self
,
inputs
)
elif
inputs
.
__class__
.
__name__
==
"OutputFileList"
and
(
inputs
.
format
==
self
.
format
or
self
.
format
==
Formats
.
ANY
):
return
list
.
__init__
(
self
,
inputs
)
elif
inputs
.
__class__
.
__name__
==
"OutputFileEndsWith"
and
(
inputs
.
format
==
self
.
format
or
self
.
format
==
Formats
.
ANY
):
return
list
.
__init__
(
self
,
inputs
)
elif
inputs
.
__class__
.
__name__
==
"OutputFilePattern"
and
(
inputs
.
format
==
self
.
format
or
self
.
format
==
Formats
.
ANY
):
return
list
.
__init__
(
self
,
inputs
)
elif
(
inputs
.
__class__
.
__name__
==
"OutputFile"
or
inputs
.
__class__
.
__name__
==
"OutputFileList"
)
and
\
not
(
inputs
.
format
==
self
.
format
or
self
.
format
==
Formats
.
ANY
):
raise
IOError
(
inputs
.
format
+
" format is incompatible with allowed format "
+
self
.
format
)
...
...
@@ -94,6 +104,19 @@ class InputDirectoryList(list):
#
# Outputs classes
#
class
DynamicOutput
(
list
):
"""
@warning : with this class of output, the component become dynamic.
"""
__metaclass__
=
abc
.
ABCMeta
@
abc
.
abstractmethod
def
update
(
self
):
"""
This method is used at the end of component execution to update output list.
"""
raise
NotImplementedError
class
OutputFile
(
str
):
def
__new__
(
self
,
output
,
format
=
Formats
.
ANY
):
...
...
@@ -105,6 +128,78 @@ class OutputFile(str):
def
check
(
self
):
return
"TODO: check output file"
class
OutputFileList
(
list
):
def
__init__
(
self
,
outputs
,
format
=
Formats
.
ANY
):
self
.
format
=
format
if
outputs
.
__class__
.
__name__
==
"str"
:
return
list
.
__init__
(
self
,
[
outputs
])
elif
outputs
.
__class__
.
__name__
==
"list"
:
return
list
.
__init__
(
self
,
outputs
)
else
:
raise
IOError
(
outputs
.
__class__
.
__name__
+
" cannot be used as OutputFileList"
)
def
check
(
self
):
return
"TODO: check output file list"
class
OutputFilePattern
(
DynamicOutput
):
def
__init__
(
self
,
output_directory
,
pattern
,
format
=
Formats
.
ANY
,
include
=
True
):
"""
@warning : with this class of output, the component become dynamic.
@param output_directory : path to the directory where outputs will be created.
@param pattern : the pattern of (a part) the file names.
@param format : the files format.
@param include : if true, the files with the pattern in file name are added into output files.
If false, the files with the pattern in file name are added into output files.
"""
self
.
output_directory
=
output_directory
self
.
pattern
=
pattern
self
.
format
=
format
self
.
include
=
include
return
list
.
__init__
(
self
,
[])
def
update
(
self
):
output_files
=
list
()
for
file
in
os
.
listdir
(
self
.
output_directory
):
if
self
.
include
and
re
.
search
(
self
.
pattern
,
file
)
is
not
None
:
output_files
.
append
(
os
.
path
.
join
(
self
.
output_directory
,
file
)
)
elif
not
self
.
include
and
re
.
search
(
self
.
pattern
,
file
)
is
None
:
output_files
.
append
(
os
.
path
.
join
(
self
.
output_directory
,
file
)
)
return
list
.
__init__
(
self
,
output_files
)
def
check
(
self
):
return
"TODO: check output file list"
class
OutputFileEndsWith
(
DynamicOutput
):
def
__init__
(
self
,
output_directory
,
end_str
,
format
=
Formats
.
ANY
,
include
=
True
):
"""
@warning : with this class of output, the component become dynamic.
@param output_directory : path to the directory where outputs will be created.
@param end_str : the end of the files names.
@param format : the files format.
@param include : if true, the files with name terminated by end_str are added into output files.
If false, the files with name not terminated by end_str are added into output files.
"""
self
.
output_directory
=
output_directory
self
.
end_str
=
end_str
self
.
format
=
format
self
.
include
=
include
return
list
.
__init__
(
self
,
[])
def
update
(
self
):
output_files
=
list
()
for
file
in
os
.
listdir
(
self
.
output_directory
):
if
file
.
endswith
(
self
.
end_str
)
and
self
.
include
:
output_files
.
append
(
os
.
path
.
join
(
self
.
output_directory
,
file
)
)
elif
not
file
.
endswith
(
self
.
end_str
)
and
not
self
.
include
:
output_files
.
append
(
os
.
path
.
join
(
self
.
output_directory
,
file
)
)
return
list
.
__init__
(
self
,
output_files
)
def
check
(
self
):
return
"TODO: check output file list"
class
OutputFileList
(
list
):
...
...
src/jflow/workflow.py
View file @
a8d1eca7
...
...
@@ -209,8 +209,11 @@ class Workflow(threading.Thread):
# add the component
self
.
components
.
append
(
cmpt_object
)
# if this one require a dynamic pipeline, execute the first part
if
cmpt_object
.
is_dynamic
:
if
cmpt_object
.
is_dynamic
()
:
self
.
_execute_weaver
()
# update outputs
for
output
in
cmpt_object
.
get_dynamic_outputs
():
output
.
update
()
return
cmpt_object
else
:
raise
ImportError
(
component_name
+
" component cannot be loaded, available components are: {0}"
.
format
(
", "
.
join
(
self
.
comp_pckg
.
keys
())))
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment