Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
genotoul-bioinfo
jflow
Commits
ced62122
Commit
ced62122
authored
Aug 12, 2014
by
Jerome Mariette
Browse files
url, browse and co OK
parent
39ee8c0f
Changes
2
Hide whitespace changes
Inline
Side-by-side
src/jflow/parameter.py
View file @
ced62122
...
...
@@ -30,31 +30,6 @@ from jflow.config_reader import JFlowConfigReader
# import custom types
from
workflows.types
import
*
# function in charge to download an url file
def
download_urlfile
(
input
):
try
:
uri_object
=
urlparse
(
input
)
opener
=
urllib2
.
urlopen
(
input
)
block_size
=
8000
jflowconf
=
JFlowConfigReader
()
tmp_directory
=
os
.
path
.
join
(
jflowconf
.
get_tmp_directory
(),
os
.
path
.
basename
(
tempfile
.
NamedTemporaryFile
().
name
))
os
.
mkdir
(
tmp_directory
)
file_path
=
os
.
path
.
join
(
tmp_directory
,
os
.
path
.
basename
(
uri_object
.
path
))
local_file
=
open
(
file_path
,
'wb'
)
if
os
.
path
.
basename
(
uri_object
.
path
)
is
not
None
and
os
.
path
.
basename
(
uri_object
.
path
)
!=
""
:
metadata
=
opener
.
info
()
file_size
=
int
(
metadata
.
getheaders
(
"Content-Length"
)[
0
])
while
True
:
buffer
=
opener
.
read
(
block_size
)
# End of download
if
not
buffer
:
break
# Parts of download
local_file
.
write
(
buffer
)
local_file
.
close
()
logging
.
getLogger
(
"types.urlfile"
).
debug
(
"URL file '{0}' successfully downloaded as: {1}"
.
format
(
input
,
file_path
))
return
[
file_path
,
True
]
except
:
return
[
input
,
False
]
# define all input type available
INPUTFILE_TYPES
=
[
"inputfile"
,
"localfile"
,
"urlfile"
,
"browsefile"
]
...
...
@@ -418,38 +393,7 @@ class InputFile(Parameter, AbstractFile):
AbstractFile
.
__init__
(
self
,
file_format
)
Parameter
.
__init__
(
self
,
name
,
help
,
flag
=
flag
,
type
=
type
,
choices
=
choices
,
required
=
required
,
group
=
group
,
display_name
=
display_name
)
"""
def _prepare_input_files(self):
# handle url inputs
if self.type.__name__ == "inputfile" or self.type.__name__ == "urlfile":
new_args[param], is_uri = self._download_urlfile(args[param])
kparams = {}
for param in parameters:
kparams[param.name] = param
new_args = args
for param in args.keys():
is_uri, is_local = False, False
# handle url inputs
if kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "urlfile":
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
new_args[param][i], is_uri = self._download_urlfile(args[param][i])
else:
new_args[param], is_uri = self._download_urlfile(args[param])
# handle upload inputs
try: is_local = os.path.isfile(args[param])
except: is_local = False
if not is_uri and not is_local and (kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "browsefile"):
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
if args[param][i]:
new_args[param][i] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param][i]))
else:
if args[param]:
new_args[param] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param]))
return new_args
"""
class
OutputFile
(
Parameter
,
AbstractFile
):
...
...
src/jflow/workflow.py
View file @
ced62122
...
...
@@ -210,6 +210,32 @@ class Workflow(threading.Thread):
attribute_value
.
type
.
excludes
[
new_group
]
=
flags2exclude
break
# function in charge to download an url file
def
_download_urlfile
(
self
,
input
):
try
:
uri_object
=
urlparse
(
input
)
opener
=
urllib2
.
urlopen
(
input
)
block_size
=
8000
jflowconf
=
JFlowConfigReader
()
tmp_directory
=
os
.
path
.
join
(
jflowconf
.
get_tmp_directory
(),
os
.
path
.
basename
(
tempfile
.
NamedTemporaryFile
().
name
))
os
.
mkdir
(
tmp_directory
)
file_path
=
os
.
path
.
join
(
tmp_directory
,
os
.
path
.
basename
(
uri_object
.
path
))
local_file
=
open
(
file_path
,
'wb'
)
if
os
.
path
.
basename
(
uri_object
.
path
)
is
not
None
and
os
.
path
.
basename
(
uri_object
.
path
)
!=
""
:
metadata
=
opener
.
info
()
file_size
=
int
(
metadata
.
getheaders
(
"Content-Length"
)[
0
])
while
True
:
buffer
=
opener
.
read
(
block_size
)
# End of download
if
not
buffer
:
break
# Parts of download
local_file
.
write
(
buffer
)
local_file
.
close
()
logging
.
getLogger
(
"types.urlfile"
).
debug
(
"URL file '{0}' successfully downloaded as: {1}"
.
format
(
input
,
file_path
))
return
[
file_path
,
True
]
except
:
return
[
input
,
False
]
def
_set_parameters
(
self
,
args
):
parameters
=
self
.
get_parameters
()
for
param
in
parameters
:
...
...
@@ -221,7 +247,14 @@ class Workflow(threading.Thread):
required
=
param
.
required
,
flag
=
param
.
flag
,
group
=
param
.
group
,
display_name
=
param
.
display_name
)
self
.
__setattr__
(
param
.
name
,
new_param
)
elif
param
.
__class__
==
InputFile
:
new_param
=
InputFile
(
param
.
name
,
param
.
help
,
file_format
=
param
.
file_format
,
default
=
args
[
param
.
name
],
type
=
param
.
type
,
# handle url inputs
default
,
is_uri
=
self
.
_download_urlfile
(
args
[
param
.
name
])
# handle upload inputs
try
:
is_local
=
os
.
path
.
isfile
(
args
[
param
])
except
:
is_local
=
False
if
not
is_uri
and
not
is_local
and
param
.
type
.
__name__
==
"inputfile"
or
param
.
type
.
__name__
==
"browsefile"
:
default
=
os
.
path
.
join
(
self
.
jflow_config_reader
.
get_tmp_directory
(),
(
args
[
param
.
name
]))
new_param
=
InputFile
(
param
.
name
,
param
.
help
,
file_format
=
param
.
file_format
,
default
=
default
,
type
=
param
.
type
,
choices
=
param
.
choices
,
required
=
param
.
required
,
flag
=
param
.
flag
,
group
=
param
.
group
,
display_name
=
param
.
display_name
)
self
.
__setattr__
(
param
.
name
,
new_param
)
...
...
@@ -230,7 +263,27 @@ class Workflow(threading.Thread):
choices
=
param
.
choices
,
required
=
param
.
required
,
flag
=
param
.
flag
,
group
=
param
.
group
,
display_name
=
param
.
display_name
)
self
.
__setattr__
(
param
.
name
,
new_param
)
elif
param
.
__class__
==
ParameterList
or
param
.
__class__
==
InputFileList
or
param
.
__class__
==
OutputFileList
:
elif
param
.
__class__
==
InputFileList
:
# handle url inputs
default_vals
,
is_uri
=
[],
False
if
args
[
param
.
name
].
__class__
.
__name__
==
"list"
:
for
path
in
args
[
param
.
name
]:
new_url
,
is_uri
=
self
.
_download_urlfile
(
path
)
if
is_uri
:
default_vals
.
append
(
new_url
)
else
:
new_url
,
is_uri
=
self
.
_download_urlfile
(
args
[
param
.
name
])
if
is_uri
:
default_vals
.
append
(
new_url
)
# handle upload inputs
try
:
is_local
=
os
.
path
.
isfile
(
args
[
param
])
except
:
is_local
=
False
if
not
is_uri
and
not
is_local
and
param
.
type
.
__name__
==
"inputfile"
or
param
.
type
.
__name__
==
"browsefile"
:
if
args
[
param
.
name
].
__class__
.
__name__
==
"list"
:
for
path
in
args
[
param
.
name
]:
default_vals
.
append
(
os
.
path
.
join
(
self
.
jflow_config_reader
.
get_tmp_directory
(),
(
path
)))
else
:
default_vals
.
append
(
os
.
path
.
join
(
self
.
jflow_config_reader
.
get_tmp_directory
(),
(
args
[
param
.
name
])))
self
.
__getattribute__
(
param
.
name
).
extend
(
default_vals
)
elif
param
.
__class__
==
ParameterList
or
param
.
__class__
==
OutputFileList
:
self
.
__getattribute__
(
param
.
name
).
extend
(
args
[
param
.
name
])
elif
param
.
__class__
==
MultiParameter
:
sub_args
=
{}
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment