Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
genotoul-bioinfo
jflow
Commits
dc242d4e
Commit
dc242d4e
authored
Aug 08, 2014
by
Jerome Mariette
Browse files
handle version 4.2.2
parent
ff940d3d
Changes
18
Hide whitespace changes
Inline
Side-by-side
application.properties
View file @
dc242d4e
...
...
@@ -15,7 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
[global]
# uncomment and set if not in the PATH
# uncomment and set if not in the PATH
, should be version >= 4.2.2
#makeflow = /usr/bin/makeflow
# batch system type: local, condor, sge, moab, cluster, wq, hadoop, mpi-queue
batch_system_type
=
local
...
...
src/jflow/workflow.py
View file @
dc242d4e
...
...
@@ -85,7 +85,8 @@ class Workflow(threading.Thread):
self
.
function
=
function
self
.
parameters_section
=
parameters_section
self
.
comp_pckg
=
self
.
_import_components
()
self
.
engine_arguments
=
''
# intruduce --log-verbose to ba able to monitor the new version of makeflow >=4.2.2
self
.
engine_arguments
=
' --log-verbose '
self
.
component_nameids_is_init
=
False
self
.
component_nameids
=
{}
# try to parse engine arguments
...
...
src/weaver/__init__.py
View file @
dc242d4e
...
...
@@ -17,4 +17,4 @@ from . import script
from
.
import
stack
from
.
import
util
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/abstraction.py
View file @
dc242d4e
...
...
@@ -61,7 +61,7 @@ class Abstraction(Dataset):
Dataset
.
__init__
(
self
)
self
.
symbol
=
self
.
nest
.
symbol
self
.
batch
=
self
.
nest
.
batch
if
collect
:
self
.
collect
=
parse_input_list
(
self
.
inputs
)
else
:
...
...
@@ -127,12 +127,12 @@ class AllPairs(Abstraction):
if
self
.
native
:
# Store inputs A and B lists as required by allpairs_master
inputs_a_file
=
next
(
self
.
nest
.
stash
)
with
open
(
inputs_a_file
,
'w
+
'
)
as
fs
:
with
open
(
inputs_a_file
,
'w'
)
as
fs
:
for
input_file
in
map
(
str
,
inputs_a
):
fs
.
write
(
input_file
+
'
\n
'
)
inputs_b_file
=
next
(
self
.
nest
.
stash
)
with
open
(
inputs_b_file
,
'w
+
'
)
as
fs
:
with
open
(
inputs_b_file
,
'w'
)
as
fs
:
for
input_file
in
map
(
str
,
inputs_b
):
fs
.
write
(
input_file
+
'
\n
'
)
...
...
@@ -352,4 +352,4 @@ class Merge(Abstraction):
SENTINEL
=
Container
(
group
=
0
)
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/compat.py
View file @
dc242d4e
...
...
@@ -68,4 +68,4 @@ def compat_next(original_class):
original_class
.
next
=
original_class
.
__next__
return
original_class
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/data.py
View file @
dc242d4e
...
...
@@ -5,7 +5,7 @@
""" Weaver data module """
from
weaver.logger
import
D_DATA
,
debug
from
weaver.stack
import
CurrentNest
from
weaver.stack
import
CurrentNest
,
CurrentScript
from
weaver.util
import
iterable
,
normalize_path
,
parse_object_list
,
\
parse_string_list
,
WeaverError
...
...
@@ -140,4 +140,4 @@ def parse_output_list(output_list=None, input_list=None):
raise
WeaverError
(
D_DATA
,
'Could not parse output argument: {0}'
.
format
(
output_list
))
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/dataset.py
View file @
dc242d4e
...
...
@@ -105,7 +105,7 @@ def cache_generation(method):
"""
@
functools
.
wraps
(
method
)
def
wrapper
(
self
):
with
open
(
self
.
cache_path
,
'w
+
'
)
as
fs
:
with
open
(
self
.
cache_path
,
'w'
)
as
fs
:
for
i
in
flatten
(
method
(
self
)):
fs
.
write
(
i
.
path
+
'
\n
'
)
yield
i
...
...
@@ -384,4 +384,4 @@ class Query(Dataset):
def
_generate
(
self
):
return
self
.
_dataset
.
_query
(
self
.
_filters
,
**
self
.
_parameters
)
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/engine.py
View file @
dc242d4e
...
...
@@ -88,11 +88,11 @@ class Makeflow(Engine):
self
.
dag_file
.
write
(
'
\t
'
.
join
([
''
,
'# SYMBOL'
,
str
(
function
)])
+
'
\n
'
)
else
:
self
.
dag_file
.
write
(
'
\t
'
.
join
([
''
,
'# SYMBOL'
,
str
(
abstraction
)])
+
'
\n
'
)
# if a symbol is provided
if
symbol
:
self
.
dag_file
.
write
(
'
\t
'
.
join
([
''
,
'#
SYMBOL
\t
'
+
symbol
])
+
'
\n
'
)
self
.
dag_file
.
write
(
'
@
SYMBOL
="
'
+
symbol
+
'"
\n
'
)
# Write environmental variables
if
options
.
local
:
self
.
dag_file
.
write
(
'
\t
@BATCH_LOCAL=1
\n
'
)
...
...
@@ -155,4 +155,4 @@ class Makeflow(Engine):
"""
raise
RuntimeError
(
'Failed to execute DAG {0} using {1}:
\n
{2}'
.
format
(
self
.
dag_path
,
self
.
path
,
e
))
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/function.py
View file @
dc242d4e
...
...
@@ -81,11 +81,13 @@ class Function(object):
if
local
:
options
.
local
=
True
if
environment
:
options
.
environment
.
update
(
environment
)
if
nest
.
batch
:
options
.
batch
=
nest
.
batch
nest
.
schedule
(
abstraction
,
self
,
command
,
list
(
inputs
)
+
list
(
includes
),
outputs
,
options
,
nest
.
symbol
)
...
...
@@ -147,7 +149,7 @@ class ScriptFunction(Function):
if
executable
is
None
:
executable
=
next
(
CurrentNest
().
stash
)
with
open
(
executable
,
'w
+
'
)
as
fs
:
with
open
(
executable
,
'w'
)
as
fs
:
fs
.
write
(
source
)
os
.
chmod
(
executable
,
0o755
)
...
...
@@ -224,6 +226,7 @@ if __name__ == '__main__':
imports
.
extend
(
function
.
func_imports
)
except
AttributeError
:
pass
if
add_path
:
add_path
=
add_path
.
extend
(
os
.
path
.
join
(
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
)),
'..'
))
else
:
...
...
@@ -232,6 +235,7 @@ if __name__ == '__main__':
path
=
""
for
apath
in
add_path
:
path
+=
"sys.path.insert(0, '"
+
apath
+
"')
\n
"
source
=
self
.
PYTHON_TEMPLATE
.
format
(
', '
.
join
(
imports
),
path
,
body
,
name
)
ScriptFunction
.
__init__
(
self
,
source
,
executable
,
cmd_format
)
...
...
@@ -301,4 +305,4 @@ class Pipeline(Function):
def
__str__
(
self
):
return
self
.
cmd_format
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/logger.py
View file @
dc242d4e
...
...
@@ -23,6 +23,7 @@ D_NEST = 'nest'
D_OPTIONS
=
'options'
D_SCRIPT
=
'script'
D_UTIL
=
'utility'
D_USER
=
'user'
D_ALL
=
'all'
...
...
@@ -119,7 +120,7 @@ class Logger(object):
self
.
log
(
'W'
,
system
,
message
)
def
set_log_path
(
self
,
path
):
self
.
stream
=
open
(
path
,
'w
+
'
)
self
.
stream
=
open
(
path
,
'w'
)
# Internal Weaver Logger instance
...
...
@@ -134,4 +135,4 @@ fatal = _WeaverLogger.fatal
warn
=
_WeaverLogger
.
warn
set_log_path
=
_WeaverLogger
.
set_log_path
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/nest.py
View file @
dc242d4e
...
...
@@ -196,7 +196,7 @@ class Nest(Makeflow):
options
,
self
))
if
abstraction
is
None
:
abstraction
=
SENTINEL
abstraction
=
SENTINEL
self
.
tasks
.
append
(
[
abstraction
,
function
,
command
,
inputs
,
outputs
,
options
,
symbol
])
...
...
@@ -229,4 +229,4 @@ def Define(key, value, export=False):
def
Export
(
variables
):
CurrentNest
().
exports
.
update
(
parse_string_list
(
variables
))
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/options.py
View file @
dc242d4e
...
...
@@ -59,4 +59,4 @@ class Options(object):
return
'Options(cpu={0}, memory={1}, disk={2}, batch={3}, local={4}, collect={5}, environment={6})'
.
format
(
self
.
cpu
,
self
.
memory
,
self
.
disk
,
self
.
batch
,
self
.
local
,
self
.
collect
,
self
.
environment
)
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/script.py
View file @
dc242d4e
...
...
@@ -47,6 +47,8 @@ class Script(object):
self
.
set_options
(
args
.
popleft
().
split
(
','
)),
'-I'
:
lambda
self
,
args
:
setattr
(
self
,
'import_builtins'
,
False
),
'-N'
:
lambda
self
,
args
:
setattr
(
self
,
'normalize_paths'
,
False
),
'-d'
:
lambda
self
,
args
:
weaver
.
logger
.
enable
(
args
.
popleft
().
split
(
','
)),
'-f'
:
lambda
self
,
args
:
...
...
@@ -58,7 +60,7 @@ class Script(object):
'-x'
:
lambda
self
,
args
:
setattr
(
self
,
'execute_dag'
,
True
),
'-o'
:
lambda
self
,
args
:
setattr
(
self
,
'output_directory'
,
os
.
path
.
abspath
(
str
(
args
.
popleft
()))
)
,
setattr
(
self
,
'output_directory'
,
str
(
args
.
popleft
())),
'-a'
:
lambda
self
,
args
:
setattr
(
self
,
'nested_abstractions'
,
True
),
'-t'
:
lambda
self
,
args
:
...
...
@@ -85,6 +87,7 @@ class Script(object):
self
.
engine_wrapper
=
None
self
.
engine_arguments
=
None
self
.
include_symbols
=
False
self
.
normalize_paths
=
True
args
=
collections
.
deque
(
args
)
while
args
:
...
...
@@ -99,6 +102,9 @@ class Script(object):
except
(
IndexError
,
KeyError
):
fatal
(
D_SCRIPT
,
'invalid command line option: {0}'
.
format
(
arg
))
if
self
.
normalize_paths
:
self
.
output_directory
=
os
.
path
.
abspath
(
self
.
output_directory
)
debug
(
D_SCRIPT
,
'path = {0}'
.
format
(
self
.
path
))
debug
(
D_SCRIPT
,
'force = {0}'
.
format
(
self
.
force
))
debug
(
D_SCRIPT
,
'import_builtins = {0}'
.
format
(
self
.
import_builtins
))
...
...
@@ -110,6 +116,7 @@ class Script(object):
debug
(
D_SCRIPT
,
'execute_dag = {0}'
.
format
(
self
.
execute_dag
))
debug
(
D_SCRIPT
,
'engine_wrapper = {0}'
.
format
(
self
.
engine_wrapper
))
debug
(
D_SCRIPT
,
'engine_arguments = {0}'
.
format
(
self
.
engine_arguments
))
debug
(
D_SCRIPT
,
'normalize_paths = {0}'
.
format
(
self
.
normalize_paths
))
if
self
.
path
is
None
:
self
.
show_usage
()
...
...
@@ -150,6 +157,7 @@ General Options:
-f Ignore warnings.
-g Include debugging symbols in DAG.
-I Do not automatically import built-ins.
-N Do not normalize paths.
-b <options> Set batch job options (cpu, memory, disk, batch, local, collect).
-d <subsystem> Enable debugging for subsystem.
-l <log_path> Set log path (default: stderr).
...
...
@@ -206,4 +214,4 @@ Subsystems:
nest
.
dag_path
,
nest
.
path
))
nest
.
execute
(
self
.
engine_arguments
,
exit_on_failure
=
True
)
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/stack.py
View file @
dc242d4e
...
...
@@ -102,4 +102,4 @@ def CurrentScript():
WeaverScripts
.
push
(
Script
(
sys
.
argv
[
0
]))
return
WeaverScripts
.
top
()
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
src/weaver/test/__init__.py
View file @
dc242d4e
...
...
@@ -4,4 +4,4 @@
""" Weaver test sub-package """
# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
\ No newline at end of file
src/weaver/test/logger.py
View file @
dc242d4e
...
...
@@ -64,4 +64,4 @@ if __name__ == '__main__':
test_suite
=
TestLoader
().
loadTestsFromTestCase
(
LoggerTestCase
)
test_runner
.
run
(
test_suite
)
# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
\ No newline at end of file
src/weaver/test/util.py
View file @
dc242d4e
...
...
@@ -89,4 +89,4 @@ if __name__ == '__main__':
test_suite
=
TestSuite
(
map
(
TestLoader
().
loadTestsFromTestCase
,
TestCases
))
test_runner
.
run
(
test_suite
)
# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
\ No newline at end of file
src/weaver/util.py
View file @
dc242d4e
...
...
@@ -200,7 +200,7 @@ def find_executable(executable, find_dirs=None):
them in `find_dirs`.
"""
if
os
.
path
.
isfile
(
executable
):
return
os
.
path
.
abs
path
(
executable
)
return
normalize_
path
(
executable
)
find_dirs
=
parse_string_list
(
find_dirs
)
find_dirs
=
itertools
.
chain
(
find_dirs
,
...
...
@@ -220,6 +220,11 @@ def normalize_path(path, ref_path=None):
If path is absolute or no `ref_path` is specified, then return absolute
path. Otherwise, return relative path.
"""
from
weaver.stack
import
CurrentScript
if
not
CurrentScript
().
normalize_paths
:
return
path
if
os
.
path
.
isabs
(
path
):
return
path
...
...
@@ -255,4 +260,4 @@ def type_str(obj, full=False):
else
:
return
obj_type
.
split
(
"'"
)[
1
].
split
(
"."
)[
-
1
]
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment