Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
genotoul-bioinfo
jflow
Commits
2db31ce3
Commit
2db31ce3
authored
May 19, 2014
by
Jerome Mariette
Browse files
improve the way workflows info are gathered ...
parent
6dc110a2
Changes
2
Hide whitespace changes
Inline
Side-by-side
src/jflow/workflow.py
View file @
2db31ce3
...
...
@@ -53,7 +53,7 @@ class Workflow(threading.Thread):
PROPERTIES_FILE_NAME
=
"workflow.properties"
MAKEFLOW_LOG_FILE_NAME
=
"Makeflow.makeflowlog"
DUMP_FILE_NAME
=
"workflow.dump"
DUMP_FILE_NAME
=
"
.
workflow.dump"
WORKING
=
".working"
STATUS_STARTED
=
"started"
STATUS_COMPLETED
=
"completed"
...
...
src/jflow/workflows_manager.py
View file @
2db31ce3
...
...
@@ -33,9 +33,11 @@ class WorkflowsManager(object):
"""
IDS_FILE_NAME
=
"jflowIDs.txt"
WORKFLOWS_DUMP_FILE_NAME
=
".workflows.dump"
WF_DIRECTORY_PREFIX
=
"wf"
lock_ids_file
=
threading
.
Lock
()
lock_workflows_file
=
threading
.
Lock
()
def
__init__
(
self
):
"""
...
...
@@ -43,8 +45,20 @@ class WorkflowsManager(object):
self
.
config_reader
=
JFlowConfigReader
()
if
not
os
.
path
.
isdir
(
self
.
get_output_directory
()):
os
.
makedirs
(
self
.
get_output_directory
(),
0751
)
self
.
dump_file
=
os
.
path
.
join
(
self
.
get_output_directory
(),
self
.
WORKFLOWS_DUMP_FILE_NAME
)
# if the file already exists, load all the workflows dump path
if
os
.
path
.
isfile
(
self
.
dump_file
):
self
.
workflows_dump
=
pickle
.
load
(
open
(
self
.
dump_file
))
else
:
self
.
workflows_dump
=
{}
self
.
ids_file
=
os
.
path
.
join
(
self
.
get_output_directory
(),
self
.
IDS_FILE_NAME
)
def
_add_workflow
(
self
,
workflow
):
WorkflowsManager
.
lock_workflows_file
.
acquire
()
self
.
workflows_dump
[
utils
.
get_nb_string
(
workflow
.
id
)]
=
workflow
.
dump_path
pickle
.
dump
(
self
.
workflows_dump
,
open
(
self
.
dump_file
,
"w"
))
WorkflowsManager
.
lock_workflows_file
.
release
()
def
get_available_workflows
(
self
,
parameters_section
=
"parameters"
):
wf_instances
=
[]
# Load all modules within the workflow module
...
...
@@ -70,26 +84,21 @@ class WorkflowsManager(object):
for
class_name
,
obj
in
inspect
.
getmembers
(
sys
.
modules
[
modname
],
inspect
.
isclass
):
if
class_name
==
workflow_class
:
workflow
=
obj
(
args
,
self
.
get_next_id
(),
function
,
parameters_section
)
workflow
.
start
()
# Add the workflow dump path to the workflows dump
self
.
_add_workflow
(
workflow
)
return
workflow
def
get_output_directory
(
self
):
return
self
.
config_reader
.
get_work_directory
()
def
get_workflows
(
self
):
from
jflow.workflow
import
Workflow
workflows
=
[]
for
wf_name
in
os
.
listdir
(
self
.
get_output_directory
()):
wf_name_path
=
os
.
path
.
join
(
self
.
get_output_directory
(),
wf_name
)
# If it's a workflow directory
if
os
.
path
.
isdir
(
wf_name_path
):
for
wf
in
os
.
listdir
(
wf_name_path
):
wf_path
=
os
.
path
.
join
(
wf_name_path
,
wf
)
workflow_dump_path
=
os
.
path
.
join
(
wf_path
,
Workflow
.
DUMP_FILE_NAME
)
try
:
workflow_dump
=
open
(
workflow_dump_path
,
"rb"
)
workflows
.
append
(
pickle
.
load
(
workflow_dump
))
workflow_dump
.
close
()
except
:
pass
for
workflow_id
in
self
.
workflows_dump
:
try
:
workflow_dump
=
open
(
self
.
workflows_dump
[
workflow_id
],
"rb"
)
workflows
.
append
(
pickle
.
load
(
workflow_dump
))
workflow_dump
.
close
()
except
:
pass
return
workflows
def
get_workflow_by_class
(
self
,
workflow_class
):
...
...
@@ -103,25 +112,13 @@ class WorkflowsManager(object):
return
None
def
get_workflow
(
self
,
workflow_id
):
from
jflow.workflow
import
Workflow
search_directory
=
self
.
WF_DIRECTORY_PREFIX
+
utils
.
get_nb_string
(
workflow_id
)
workflow_dump_path
=
None
for
wf_name
in
os
.
listdir
(
self
.
get_output_directory
()):
wf_name_path
=
os
.
path
.
join
(
self
.
get_output_directory
(),
wf_name
)
# If it's a workflow directory
if
os
.
path
.
isdir
(
wf_name_path
):
for
wf
in
os
.
listdir
(
wf_name_path
):
wf_path
=
os
.
path
.
join
(
wf_name_path
,
wf
)
if
wf
==
search_directory
:
if
os
.
path
.
isfile
(
os
.
path
.
join
(
wf_path
,
Workflow
.
DUMP_FILE_NAME
)):
workflow_dump_path
=
os
.
path
.
join
(
wf_path
,
Workflow
.
DUMP_FILE_NAME
)
break
if
workflow_dump_path
:
workflow_dump
=
open
(
workflow_dump_path
,
"rb"
)
rworkflow_id
=
utils
.
get_nb_string
(
workflow_id
)
if
self
.
workflows_dump
.
has_key
(
rworkflow_id
):
workflow_dump
=
open
(
self
.
workflows_dump
[
rworkflow_id
],
"rb"
)
workflow
=
pickle
.
load
(
workflow_dump
)
workflow_dump
.
close
()
else
:
raise
ValueError
(
"Workflow with id "
+
str
(
workflow_id
)
+
" cannot be retrived!"
)
raise
ValueError
(
"Workflow with id "
+
str
(
r
workflow_id
)
+
" cannot be retrived!"
)
return
workflow
def
get_workflow_directory
(
self
,
wname
,
wid
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment