Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
genotoul-bioinfo
D-GENIES
Commits
7f5a4b2f
Commit
7f5a4b2f
authored
Nov 27, 2017
by
Floreal Cabanettes
Browse files
Some code refactoring
parent
001ed535
Changes
11
Hide whitespace changes
Inline
Side-by-side
bin/clean_jobs.py
View file @
7f5a4b2f
...
...
@@ -18,8 +18,8 @@ from database import Job
config_reader
=
AppConfigReader
()
UPLOAD_FOLDER
=
config_reader
.
get_
upload_folder
()
APP_DATA
=
config_reader
.
get_
app_data
()
UPLOAD_FOLDER
=
config_reader
.
upload_folder
APP_DATA
=
config_reader
.
app_data
NOW
=
time
.
time
()
FAKE
=
False
...
...
bin/local_scheduler.py
View file @
7f5a4b2f
#!/usr/bin/env python3
import
os
import
threading
import
time
import
sys
import
psutil
...
...
@@ -20,7 +19,7 @@ from config_reader import AppConfigReader
from
lib.job_manager
import
JobManager
config_reader
=
AppConfigReader
()
NB_RUN
=
config_reader
.
get_
local_nb_runs
()
NB_RUN
=
config_reader
.
local_nb_runs
@
db_session
...
...
@@ -32,9 +31,6 @@ def start_job(id_job):
job_mng
=
JobManager
(
id_job
=
id_job
,
email
=
job
.
email
)
job_mng
.
set_inputs_from_res_dir
()
job_mng
.
run_job_in_thread
()
# thread = threading.Timer(1, job_mng.start_job, kwargs={"batch_system_type": "local"})
# thread.daemon = True # Daemonize thread
# thread.start() # Start the execution
@
db_session
...
...
bin/sort_paf.py
View file @
7f5a4b2f
...
...
@@ -32,7 +32,7 @@ def __sort_key_paf_lines(a):
return
-
sqrt
(
pow
(
x2
-
x1
,
2
)
+
pow
(
y2
-
y1
,
2
))
*
(
int
(
a
[
9
])
/
int
(
a
[
10
]))
def
__get_sorted_paf_lines
(
lines
):
def
__get_sorted_paf_lines
(
lines
:
iter
):
paf_lines
=
[]
for
line
in
lines
:
parts
=
line
.
strip
(
"
\n
"
).
split
(
"
\t
"
)
...
...
@@ -43,8 +43,7 @@ def __get_sorted_paf_lines(lines):
def
init
(
input_f
,
output_f
):
with
open
(
input_f
,
"r"
)
as
paf_file
:
paf_lines
=
paf_file
.
readlines
()
paf_lines
=
__get_sorted_paf_lines
(
paf_lines
)
paf_lines
=
__get_sorted_paf_lines
(
paf_file
)
with
open
(
output_f
,
"w"
)
as
out
:
out
.
write
(
"
\n
"
.
join
([
"
\t
"
.
join
(
x
)
for
x
in
paf_lines
]))
...
...
lib/crons.py
View file @
7f5a4b2f
...
...
@@ -11,7 +11,7 @@ class Crons:
def
__init__
(
self
,
base_dir
):
self
.
base_dir
=
base_dir
self
.
my_cron
=
CronTab
(
user
=
getpass
.
getuser
())
self
.
config
_reader
=
AppConfigReader
()
self
.
config
=
AppConfigReader
()
self
.
local_scheduler_pid_file
=
os
.
path
.
join
(
self
.
base_dir
,
".local_scheduler_pid"
)
def
clear
(
self
,
kill_scheduler
=
True
):
...
...
@@ -36,8 +36,8 @@ class Crons:
"""
Menage cron is launched at 1h00am each day
"""
menage_hour
=
self
.
config
_reader
.
get_
cron_menage_hour
()
menage_freq
=
self
.
config
_reader
.
get_
cron_menage_freq
()
menage_hour
=
self
.
config
.
cron_menage_hour
menage_freq
=
self
.
config
.
cron_menage_freq
if
self
.
base_dir
is
not
None
:
job
=
self
.
my_cron
.
new
(
sys
.
executable
+
" {0}/bin/clean_jobs.py > {0}/logs/menage.log 2>&1"
.
format
(
self
.
base_dir
),
...
...
lib/functions.py
View file @
7f5a4b2f
...
...
@@ -165,8 +165,8 @@ class Functions:
@
staticmethod
def
send_fasta_ready
(
mailer
,
job_name
,
sample_name
,
compressed
=
False
):
config
_reader
=
AppConfigReader
()
web_url
=
config
_reader
.
get_
web_url
()
config
=
AppConfigReader
()
web_url
=
config
.
web_url
with
open
(
os
.
path
.
join
(
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
)),
"mail_templates"
,
"dl_fasta.html"
))
\
as
t_file
:
template
=
Template
(
t_file
.
read
())
...
...
lib/job_manager.py
View file @
7f5a4b2f
...
...
@@ -24,20 +24,11 @@ class JobManager:
self
.
email
=
email
self
.
query
=
query
self
.
target
=
target
config_reader
=
AppConfigReader
()
self
.
error
=
""
# Get configs:
self
.
batch_system_type
=
config_reader
.
get_batch_system_type
()
self
.
minimap2
=
config_reader
.
get_minimap2_exec
()
self
.
threads
=
config_reader
.
get_nb_threads
()
self
.
app_data
=
config_reader
.
get_app_data
()
self
.
web_url
=
config_reader
.
get_web_url
()
self
.
mail_status
=
config_reader
.
get_mail_status_sender
()
self
.
mail_reply
=
config_reader
.
get_mail_reply
()
self
.
mail_org
=
config_reader
.
get_mail_org
()
self
.
do_send
=
config_reader
.
get_send_mail_status
()
self
.
config
=
AppConfigReader
()
# Outputs:
self
.
output_dir
=
os
.
path
.
join
(
self
.
app_data
,
id_job
)
self
.
output_dir
=
os
.
path
.
join
(
self
.
config
.
app_data
,
id_job
)
self
.
paf
=
os
.
path
.
join
(
self
.
output_dir
,
"map.paf"
)
self
.
paf_raw
=
os
.
path
.
join
(
self
.
output_dir
,
"map_raw.paf"
)
self
.
idx_q
=
os
.
path
.
join
(
self
.
output_dir
,
"query.idx"
)
...
...
@@ -46,7 +37,7 @@ class JobManager:
self
.
mailer
=
mailer
def
set_inputs_from_res_dir
(
self
):
res_dir
=
os
.
path
.
join
(
self
.
app_data
,
self
.
id_job
)
res_dir
=
os
.
path
.
join
(
self
.
config
.
app_data
,
self
.
id_job
)
query_file
=
os
.
path
.
join
(
res_dir
,
".query"
)
if
os
.
path
.
exists
(
query_file
):
with
open
(
query_file
)
as
q_f
:
...
...
@@ -75,7 +66,7 @@ class JobManager:
return
"fail"
def
check_job_success
(
self
):
if
self
.
batch_system_type
==
"local"
:
if
self
.
config
.
batch_system_type
==
"local"
:
return
self
.
__check_job_success_local
()
def
get_mail_content
(
self
,
status
):
...
...
@@ -83,7 +74,7 @@ class JobManager:
if
status
==
"success"
:
message
+=
"Your job %s was completed successfully!
\n\n
"
%
self
.
id_job
message
+=
str
(
"Your job {0} is finished. You can see the results by clicking on the link below:
\n
"
"{1}/result/{0}
\n\n
"
).
format
(
self
.
id_job
,
self
.
web_url
)
"{1}/result/{0}
\n\n
"
).
format
(
self
.
id_job
,
self
.
config
.
web_url
)
else
:
message
+=
"Your job %s has failed!
\n\n
"
%
self
.
id_job
if
self
.
error
!=
""
:
...
...
@@ -104,7 +95,7 @@ class JobManager:
with
open
(
os
.
path
.
join
(
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
)),
"mail_templates"
,
"job_notification.html"
))
\
as
t_file
:
template
=
Template
(
t_file
.
read
())
return
template
.
render
(
job_name
=
self
.
id_job
,
status
=
status
,
url_base
=
self
.
web_url
,
return
template
.
render
(
job_name
=
self
.
id_job
,
status
=
status
,
url_base
=
self
.
config
.
web_url
,
query_name
=
self
.
query
.
get_name
()
if
self
.
query
is
not
None
else
""
,
target_name
=
self
.
target
.
get_name
(),
error
=
self
.
error
)
...
...
@@ -142,7 +133,7 @@ class JobManager:
@
db_session
def
__launch_local
(
self
):
cmd
=
[
"run_minimap2.sh"
,
self
.
minimap2
,
self
.
threads
,
self
.
target
.
get_path
(),
cmd
=
[
"run_minimap2.sh"
,
self
.
config
.
minimap2
_exec
,
self
.
config
.
nb_
threads
,
self
.
target
.
get_path
(),
self
.
query
.
get_path
()
if
self
.
query
is
not
None
else
"NONE"
,
self
.
paf
,
self
.
paf_raw
]
with
open
(
self
.
logs
,
"w"
)
as
logs
:
p
=
subprocess
.
Popen
(
cmd
,
stdout
=
logs
,
stderr
=
logs
)
...
...
@@ -236,11 +227,11 @@ class JobManager:
Send mail using POST url (we have no access to mailer)
"""
key
=
Functions
.
random_string
(
15
)
key_file
=
os
.
path
.
join
(
self
.
app_data
,
self
.
id_job
,
".key"
)
key_file
=
os
.
path
.
join
(
self
.
config
.
app_data
,
self
.
id_job
,
".key"
)
with
open
(
key_file
,
"w"
)
as
k_f
:
k_f
.
write
(
key
)
data
=
parse
.
urlencode
({
"key"
:
key
}).
encode
()
req
=
request
.
Request
(
self
.
web_url
+
"/send-mail/"
+
self
.
id_job
,
data
=
data
)
req
=
request
.
Request
(
self
.
config
.
web_url
+
"/send-mail/"
+
self
.
id_job
,
data
=
data
)
resp
=
request
.
urlopen
(
req
)
if
resp
.
getcode
()
!=
200
:
print
(
"Job %s: Send mail failed!"
%
self
.
id_job
)
...
...
@@ -271,9 +262,8 @@ class JobManager:
job
=
Job
.
get
(
id_job
=
self
.
id_job
)
job
.
status
=
"success"
db
.
commit
()
if
self
.
do_send
:
if
self
.
config
.
send_mail_status
:
self
.
send_mail_post
()
#self.send_mail(job.status)
@
db_session
def
start_job
(
self
):
...
...
@@ -288,7 +278,7 @@ class JobManager:
job
.
status
=
"fail"
job
.
error
=
"<p>Error while getting input files. Please contact the support to report the bug.</p>"
db
.
commit
()
if
self
.
do_send
:
if
self
.
config
.
send_mail_status
:
self
.
send_mail
()
except
Exception
:
...
...
@@ -297,7 +287,7 @@ class JobManager:
job
.
status
=
"fail"
job
.
error
=
"<p>An unexpected error has occurred. Please contact the support to report the bug.</p>"
db
.
commit
()
if
self
.
do_send
:
if
self
.
config
.
send_mail_status
:
self
.
send_mail
()
...
...
@@ -308,7 +298,7 @@ class JobManager:
print
(
"Old job found without result dir existing: delete it from BDD!"
)
j1
.
delete
()
if
self
.
target
is
not
None
:
job
=
Job
(
id_job
=
self
.
id_job
,
email
=
self
.
email
,
batch_type
=
self
.
batch_system_type
,
job
=
Job
(
id_job
=
self
.
id_job
,
email
=
self
.
email
,
batch_type
=
self
.
config
.
batch_system_type
,
date_created
=
datetime
.
datetime
.
now
())
db
.
commit
()
if
not
os
.
path
.
exists
(
self
.
output_dir
):
...
...
@@ -316,7 +306,7 @@ class JobManager:
thread
=
threading
.
Timer
(
1
,
self
.
start_job
)
thread
.
start
()
else
:
job
=
Job
(
id_job
=
self
.
id_job
,
email
=
self
.
email
,
batch_type
=
self
.
batch_system_type
,
job
=
Job
(
id_job
=
self
.
id_job
,
email
=
self
.
email
,
batch_type
=
self
.
config
.
batch_system_type
,
date_created
=
datetime
.
datetime
.
now
(),
status
=
"fail"
)
db
.
commit
()
...
...
lib/mailer.py
View file @
7f5a4b2f
...
...
@@ -7,20 +7,21 @@ class Mailer:
def
__init__
(
self
,
app
):
self
.
app
=
app
self
.
mail
=
Mail
(
app
)
config
_reader
=
AppConfigReader
()
self
.
mail_status
=
config_reader
.
get_mail_status_sender
()
self
.
mail_reply
=
config_reader
.
get_mail_reply
()
self
.
mail_org
=
config_reader
.
get_mail_org
()
self
.
disable
=
config_reader
.
get_disable_mail
()
self
.
config
=
AppConfigReader
()
#
self.mail_status = config_reader.get_mail_status_sender()
#
self.mail_reply = config_reader.get_mail_reply()
#
self.mail_org = config_reader.get_mail_org()
#
self.disable = config_reader.get_disable_mail()
def
__send_async_email
(
self
,
msg
):
with
self
.
app
.
app_context
():
self
.
mail
.
send
(
msg
)
def
send_mail
(
self
,
recipients
:
list
,
subject
:
str
,
message
:
str
,
message_html
:
str
=
None
):
sender
=
(
self
.
mail_org
,
self
.
mail_status
)
if
self
.
mail_org
is
not
None
else
self
.
mail_status
reply
=
self
.
mail_reply
if
not
self
.
disable
:
sender
=
(
self
.
config
.
mail_org
,
self
.
config
.
mail_status_sender
)
if
self
.
config
.
mail_org
is
not
None
else
\
self
.
config
.
mail_status_sender
reply
=
self
.
config
.
mail_reply
if
not
self
.
config
.
disable_mail
:
msg
=
Message
(
subject
=
subject
,
recipients
=
recipients
,
...
...
@@ -36,7 +37,7 @@ class Mailer:
"################
\n\n
"
"!!! SEND MAILS DISABLED BY CONFIGURATION !!!
\n\n
"
"(This might be disabled in production)
\n\n
"
)
print
(
"Sender: %s <%s>
\n
"
%
sender
)
print
(
"Sender: %s <%s>
\n
"
%
sender
if
isinstance
(
tuple
,
sender
)
else
(
"None"
,
sender
)
)
print
(
"Reply to: %s
\n
"
%
reply
)
print
(
"Recipients: %s
\n
"
%
", "
.
join
(
recipients
))
print
(
"Subject: %s
\n
"
%
subject
)
...
...
lib/paf.py
View file @
7f5a4b2f
...
...
@@ -141,10 +141,12 @@ class Paf:
try
:
with
open
(
self
.
paf
,
"r"
)
as
paf_file
:
paf_lines
=
paf_file
.
readlines
()
if
len
(
paf_lines
)
>
self
.
max_nb_lines
:
self
.
sampled
=
True
for
line
in
paf_lines
[:
self
.
max_nb_lines
]:
nb_lines
=
0
for
line
in
paf_file
:
nb_lines
+=
1
if
nb_lines
>
self
.
max_nb_lines
:
self
.
sampled
=
True
break
parts
=
line
.
strip
(
"
\n
"
).
split
(
"
\t
"
)
v1
=
parts
[
0
]
v6
=
parts
[
5
]
...
...
srv/config_reader.py
View file @
7f5a4b2f
...
...
@@ -2,16 +2,21 @@ import os
import
re
import
inspect
from
configparser
import
RawConfigParser
,
NoOptionError
,
NoSectionError
from
lib.decorators
import
Singleton
class
AppConfigReader
(
object
):
@
Singleton
class
AppConfigReader
:
"""
Store all configs
"""
CONFIG_FILE_PATH
=
"../application.properties"
def
__init__
(
self
):
"""
All "get_*" functions results are stored in the "self.*" corresponding attribute
Example: results of the get_upload_folder function is stored in self.upload_folder
"""
config_file
=
os
.
path
.
join
(
os
.
path
.
dirname
(
inspect
.
getfile
(
self
.
__class__
)),
self
.
CONFIG_FILE_PATH
)
if
not
os
.
path
.
exists
(
config_file
):
...
...
@@ -19,6 +24,10 @@ class AppConfigReader(object):
"properties are correct for you!"
)
self
.
reader
=
RawConfigParser
()
self
.
reader
.
read
(
config_file
)
for
attr
in
dir
(
self
):
attr_o
=
getattr
(
self
,
attr
)
if
attr
.
startswith
(
"get_"
)
and
callable
(
attr_o
):
setattr
(
self
,
attr
[
4
:],
attr_o
())
@
staticmethod
def
replace_vars
(
path
):
...
...
srv/database.py
View file @
7f5a4b2f
...
...
@@ -2,8 +2,8 @@ import datetime
from
config_reader
import
AppConfigReader
from
pony.orm
import
Database
,
Required
,
Optional
config
_reader
=
AppConfigReader
()
file_path
=
config
_reader
.
get_
database
()
config
=
AppConfigReader
()
file_path
=
config
.
database
db
=
Database
()
db
.
bind
(
provider
=
'sqlite'
,
filename
=
file_path
,
create_db
=
True
)
...
...
srv/main.py
View file @
7f5a4b2f
...
...
@@ -29,8 +29,8 @@ sqlite_file = os.path.join(app_folder, "database.sqlite")
# Init config reader:
config_reader
=
AppConfigReader
()
UPLOAD_FOLDER
=
config_reader
.
get_
upload_folder
()
APP_DATA
=
config_reader
.
get_
app_data
()
UPLOAD_FOLDER
=
config_reader
.
upload_folder
APP_DATA
=
config_reader
.
app_data
app_title
=
"D-GENIES - Dotplot for Genomes Interactive, E-connected and Speedy"
...
...
@@ -43,7 +43,7 @@ app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
mailer
=
Mailer
(
app
)
# Folder containing data:
app_data
=
config_reader
.
get_
app_data
()
app_data
=
config_reader
.
app_data
# Crons:
crons
=
Crons
(
app_folder
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment