Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Maintenance - Mise à jour mensuelle Lundi 6 Février entre 7h00 et 9h00
Open sidebar
genotoul-bioinfo
D-GENIES
Commits
bd2b2cf2
Commit
bd2b2cf2
authored
Feb 15, 2018
by
Floreal Cabanettes
Browse files
Remove wget requirement + add debug print on failure
parent
8bb77227
Changes
3
Hide whitespace changes
Inline
Side-by-side
requirements.txt
View file @
bd2b2cf2
...
...
@@ -3,7 +3,6 @@ Flask-Mail==0.9.*
Jinja2
==2.9.*
peewee
==2.10.2
numpy
wget
==3.2
requests
==2.18.*
biopython
==1.70
python-crontab
==2.2.*
...
...
src/dgenies/INSTALL.md
View file @
bd2b2cf2
...
...
@@ -45,7 +45,6 @@ Some python modules are required (will be automatically installed by the command
flask==0.12.*
Jinja2==2.9.*
numpy
wget==3.2
requests==2.18.*
biopython==1.70
psutil==5.4.*
...
...
src/dgenies/lib/job_manager.py
View file @
bd2b2cf2
...
...
@@ -12,7 +12,8 @@ from .fasta import Fasta
from
.functions
import
Functions
import
requests
from
requests.exceptions
import
ConnectionError
import
wget
from
urllib.request
import
urlretrieve
from
urllib.error
import
URLError
from
jinja2
import
Template
import
traceback
from
pathlib
import
Path
...
...
@@ -54,6 +55,7 @@ class JobManager:
self
.
idx_t
=
os
.
path
.
join
(
self
.
output_dir
,
"target.idx"
)
self
.
logs
=
os
.
path
.
join
(
self
.
output_dir
,
"logs.txt"
)
self
.
mailer
=
mailer
self
.
_filename_for_url
=
{}
@
staticmethod
def
is_gz_file
(
filepath
):
...
...
@@ -374,6 +376,30 @@ class JobManager:
save_file
.
write
(
finale_path
)
return
finale_path
def
__get_filename_from_url
(
self
,
url
):
if
url
not
in
self
.
_filename_for_url
:
if
url
.
startswith
(
"ftp://"
):
self
.
_filename_for_url
[
url
]
=
url
.
split
(
"/"
)[
-
1
]
elif
url
.
startswith
(
"http://"
)
or
url
.
startswith
(
"https://"
):
self
.
_filename_for_url
[
url
]
=
requests
.
head
(
url
,
allow_redirects
=
True
).
url
.
split
(
"/"
)[
-
1
]
else
:
return
None
return
self
.
_filename_for_url
[
url
]
def
_download_file
(
self
,
url
):
local_filename
=
os
.
path
.
join
(
self
.
output_dir
,
self
.
__get_filename_from_url
(
url
))
# NOTE the stream=True parameter
if
url
.
startswith
(
"ftp://"
):
urlretrieve
(
url
,
local_filename
)
else
:
r
=
requests
.
get
(
url
,
stream
=
True
)
with
open
(
local_filename
,
'wb'
)
as
f
:
for
chunk
in
r
.
iter_content
(
chunk_size
=
1024
):
if
chunk
:
# filter out keep-alive new chunks
f
.
write
(
chunk
)
# f.flush() commented by recommendation from J.F.Sebastian
return
local_filename
def
__getting_file_from_url
(
self
,
fasta
:
Fasta
,
type_f
):
"""
Download file from URL
...
...
@@ -386,8 +412,8 @@ class JobManager:
[3] Name of the downloaded file {str}
"""
try
:
dl_path
=
wget
.
download
(
fasta
.
get_path
()
,
self
.
output_dir
,
None
)
except
ConnectionError
:
dl_path
=
self
.
_
download
_file
(
fasta
.
get_path
())
except
(
ConnectionError
,
URLError
)
:
status
=
"fail"
error
=
"<p>Url <b>%s</b> is not valid!</p>"
\
"<p>If this is unattended, please contact the support.</p>"
%
fasta
.
get_path
()
...
...
@@ -410,26 +436,21 @@ class JobManager:
def
__check_url
(
self
,
fasta
:
Fasta
):
url
=
fasta
.
get_path
()
if
url
.
startswith
(
"http://"
)
or
url
.
startswith
(
"https://"
):
try
:
filename
=
requests
.
head
(
url
,
allow_redirects
=
True
).
url
.
split
(
"/"
)[
-
1
]
except
ConnectionError
:
status
=
"fail"
error
=
"<p>Url <b>%s</b> is not valid!</p>"
\
"<p>If this is unattended, please contact the support.</p>"
%
fasta
.
get_path
()
if
MODE
==
"webserver"
:
with
Job
.
connect
():
job
=
Job
.
get
(
Job
.
id_job
==
self
.
id_job
)
job
.
status
=
status
job
.
error
=
error
job
.
save
()
else
:
self
.
set_status_standalone
(
status
,
error
)
return
False
elif
url
.
startswith
(
"ftp://"
):
filename
=
url
.
split
(
"/"
)[
-
1
]
else
:
filename
=
None
try
:
filename
=
self
.
__get_filename_from_url
(
url
)
except
(
ConnectionError
,
URLError
):
status
=
"fail"
error
=
"<p>Url <b>%s</b> is not valid!</p>"
\
"<p>If this is unattended, please contact the support.</p>"
%
fasta
.
get_path
()
if
MODE
==
"webserver"
:
with
Job
.
connect
():
job
=
Job
.
get
(
Job
.
id_job
==
self
.
id_job
)
job
.
status
=
status
job
.
error
=
error
job
.
save
()
else
:
self
.
set_status_standalone
(
status
,
error
)
return
False
if
filename
is
not
None
:
allowed
=
Functions
.
allowed_file
(
filename
)
if
not
allowed
:
...
...
@@ -565,6 +586,7 @@ class JobManager:
else
:
correct
=
False
except
:
# Except all possible exceptions
traceback
.
print_exc
()
correct
=
False
error_set
=
False
if
MODE
==
"webserver"
:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment