Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
genotoul-bioinfo
D-GENIES
Commits
6c6de5f1
Commit
6c6de5f1
authored
Jan 30, 2018
by
Floreal Cabanettes
Browse files
Add gallery, Implements
#87
parent
f2cadc7d
Changes
7
Hide whitespace changes
Inline
Side-by-side
src/bin/dgenies
View file @
6c6de5f1
...
...
@@ -9,9 +9,13 @@ import time
from
dgenies.lib.crons
import
Crons
from
dgenies.config_reader
import
AppConfigReader
from
dgenies.bin.clean_jobs
import
parse_data_folders
,
parse_database
,
parse_upload_folders
from
dgenies.database
import
Gallery
,
Job
from
peewee
import
DoesNotExist
runned
=
False
config
=
AppConfigReader
()
def
parse_args
():
parser
=
argparse
.
ArgumentParser
(
description
=
"Manage dgenies application"
)
...
...
@@ -42,6 +46,28 @@ def parse_args():
clear
.
add_argument
(
"-m"
,
"--max-age"
,
help
=
"Max age for job to delete (0 for all)"
,
type
=
int
,
required
=
False
,
default
=
0
)
# Gallery:
gallery
=
subparsers
.
add_parser
(
"gallery"
,
help
=
"Manage gallery"
)
subparsers_gallery
=
gallery
.
add_subparsers
()
# Gallery add:
gallery_add
=
subparsers_gallery
.
add_parser
(
"add"
,
help
=
"Add new job to the gallery"
)
gallery_add
.
set_defaults
(
which
=
"add"
)
gallery_add
.
add_argument
(
"-i"
,
"--id-job"
,
type
=
str
,
required
=
True
,
help
=
"Id (name) of the job to add to the gallery"
)
gallery_add
.
add_argument
(
"-n"
,
"--name"
,
type
=
str
,
required
=
True
,
help
=
"Name to show in the gallery for the job"
)
gallery_add
.
add_argument
(
"-p"
,
"--pict"
,
type
=
str
,
required
=
True
,
help
=
"Name of the file that illustrate the job"
)
# Gallery del:
gallery_del
=
subparsers_gallery
.
add_parser
(
"del"
,
help
=
"Delete a job from the gallery"
)
gallery_del
.
set_defaults
(
which
=
"del"
)
gallery_del
.
add_argument
(
"-i"
,
"--id-job"
,
type
=
str
,
required
=
False
,
help
=
"Id (name) of the job to delete from the gallery"
)
gallery_del
.
add_argument
(
"-n"
,
"--name"
,
type
=
str
,
required
=
False
,
help
=
"Name of the job shown in the gallery"
)
gallery_del
.
add_argument
(
"--remove-pict"
,
help
=
"Clear crons"
,
type
=
bool
,
const
=
True
,
nargs
=
"?"
,
required
=
False
,
default
=
False
)
args
=
parser
.
parse_args
()
if
args
.
subparser_name
==
"run"
:
...
...
@@ -51,6 +77,8 @@ def parse_args():
print
(
"Nothing to do."
)
return
"clear"
,
None
return
"clear"
,
args
if
args
.
subparser_name
==
"gallery"
:
return
"gallery_"
+
args
.
which
,
args
def
start_browser
(
host
,
port
):
...
...
@@ -79,8 +107,6 @@ def clear_crons():
def
clear_logs
():
config
=
AppConfigReader
()
if
hasattr
(
config
,
"log_dir"
):
log_files
=
glob
(
os
.
path
.
join
(
config
.
log_dir
,
"*.log"
))
for
file
in
log_files
:
...
...
@@ -91,10 +117,9 @@ def clear_logs():
def
clear_jobs
(
max_data_age
=
7
):
config_reader
=
AppConfigReader
()
upload_folder
=
config
_reader
.
upload_folder
app_data
=
config
_reader
.
app_data
upload_folder
=
config
.
upload_folder
app_data
=
config
.
app_data
now
=
time
.
time
()
max_age
=
{
...
...
@@ -119,7 +144,7 @@ def clear_jobs(max_data_age=7):
print
(
"# Parsing Jobs in DB #"
)
print
(
"######################"
)
print
(
""
)
parse_database
(
gallery_jobs
=
parse_database
(
app_data
=
app_data
,
max_age
=
max_age
)
...
...
@@ -132,11 +157,44 @@ def clear_jobs(max_data_age=7):
parse_data_folders
(
app_data
=
app_data
,
now
=
now
,
max_age
=
max_age
max_age
=
max_age
,
gallery_jobs
=
gallery_jobs
)
print
(
""
)
def
add_to_gallery
(
id_job
,
name
,
picture
):
try
:
job
=
Job
.
get
(
id_job
=
id_job
)
except
DoesNotExist
:
print
(
"Error: job
\"
%s
\"
does not exists!"
%
id_job
)
exit
(
1
)
pict_file
=
os
.
path
.
join
(
config
.
app_data
,
"gallery"
,
picture
)
if
not
os
.
path
.
exists
(
pict_file
):
print
(
"Error: file
\"
%s
\"
does not exists!"
%
pict_file
)
exit
(
1
)
item
=
Gallery
.
create
(
job
=
job
,
name
=
name
,
picture
=
picture
)
item
.
save
()
def
del_from_gallery_by_id
(
id_job
):
items
=
Gallery
.
select
().
join
(
Job
).
where
(
Job
.
id_job
==
id_job
)
list_pictures
=
[]
for
item
in
items
:
list_pictures
.
append
(
item
.
picture
)
item
.
delete_instance
()
return
list_pictures
def
del_from_gallery_by_name
(
name
):
items
=
Gallery
.
select
().
where
(
Gallery
.
name
==
name
)
list_pictures
=
[]
for
item
in
items
:
list_pictures
.
append
(
item
.
picture
)
item
.
delete_instance
()
return
list_pictures
if
__name__
==
"__main__"
:
command
,
args
=
parse_args
()
if
command
==
"run"
:
...
...
@@ -151,3 +209,19 @@ if __name__ == "__main__":
if
args
.
jobs
:
print
(
"Cleaning jobs..."
)
clear_jobs
(
args
.
max_age
)
elif
command
==
"gallery_add"
:
add_to_gallery
(
args
.
id_job
,
args
.
name
,
args
.
pict
)
elif
command
==
"gallery_del"
:
if
args
.
id_job
is
None
and
args
.
name
is
None
:
print
(
"Error: please give an id or a name for the job!"
)
exit
(
1
)
if
args
.
id_job
is
not
None
:
pictures
=
del_from_gallery_by_id
(
args
.
id_job
)
else
:
pictures
=
del_from_gallery_by_name
(
args
.
name
)
if
args
.
remove_pict
:
for
picture
in
pictures
:
try
:
os
.
remove
(
os
.
path
.
join
(
config
.
app_data
,
"gallery"
,
picture
))
except
FileNotFoundError
:
pass
src/dgenies/bin/clean_jobs.py
View file @
6c6de5f1
...
...
@@ -10,7 +10,7 @@ import argparse
from
dgenies.config_reader
import
AppConfigReader
from
dgenies.lib.functions
import
Functions
from
dgenies.database
import
Job
from
dgenies.database
import
Job
,
Gallery
config_reader
=
AppConfigReader
()
...
...
@@ -35,6 +35,7 @@ def parse_upload_folders(upload_folder, now, max_age, fake=False):
def
parse_database
(
app_data
,
max_age
,
fake
=
False
):
gallery_jobs
=
[]
with
Job
.
connect
():
old_jobs
=
Job
.
select
().
where
(
((
Job
.
status
==
"success"
)
&
(
Job
.
date_created
<
datetime
.
now
()
-
timedelta
(
days
=
max_age
[
"data"
])))
...
...
@@ -43,6 +44,9 @@ def parse_database(app_data, max_age, fake=False):
)
for
job
in
old_jobs
:
id_job
=
job
.
id_job
is_gallery
=
len
(
Gallery
.
select
().
join
(
Job
).
where
(
Job
.
id_job
==
id_job
))
>
0
if
is_gallery
:
gallery_jobs
.
append
(
id_job
)
print
(
"Removing job %s..."
%
id_job
)
data_dir
=
os
.
path
.
join
(
app_data
,
id_job
)
if
os
.
path
.
exists
(
data_dir
)
and
os
.
path
.
isdir
(
data_dir
):
...
...
@@ -52,39 +56,41 @@ def parse_database(app_data, max_age, fake=False):
print
(
"Job %s has no data folder!"
%
id_job
)
if
not
fake
:
job
.
delete_instance
()
return
gallery_jobs
def
parse_data_folders
(
app_data
,
now
,
max_age
,
fake
=
False
):
def
parse_data_folders
(
app_data
,
now
,
max_age
,
gallery_jobs
,
fake
=
False
):
for
file
in
os
.
listdir
(
app_data
):
file
=
os
.
path
.
join
(
app_data
,
file
)
create_date
=
os
.
path
.
getctime
(
file
)
age
=
(
now
-
create_date
)
/
86400
# Age in days
if
age
>
max_age
[
"data"
]:
try
:
if
os
.
path
.
isdir
(
file
):
print
(
"Removing folder %s..."
%
file
)
if
not
fake
:
shutil
.
rmtree
(
file
)
else
:
print
(
"Removing file %s..."
%
file
)
if
not
fake
:
os
.
remove
(
file
)
except
OSError
:
print
(
traceback
.
print_exc
())
elif
os
.
path
.
isdir
(
file
):
query_name_file
=
os
.
path
.
join
(
file
,
".query"
)
if
os
.
path
.
exists
(
query_name_file
):
with
open
(
query_name_file
)
as
query_file
:
sorted_file
=
Functions
.
get_fasta_file
(
file
,
"query"
,
True
)
if
not
sorted_file
.
endswith
(
".sorted"
):
sorted_file
=
None
if
sorted_file
is
not
None
:
create_date
=
os
.
path
.
getctime
(
sorted_file
)
age
=
(
now
-
create_date
)
/
86400
# Age in days
if
age
>
max_age
[
"fasta_sorted"
]:
print
(
"Removing fasta file %s..."
%
sorted_file
)
if
not
fake
:
os
.
remove
(
sorted_file
)
if
file
not
in
gallery_jobs
:
file
=
os
.
path
.
join
(
app_data
,
file
)
create_date
=
os
.
path
.
getctime
(
file
)
age
=
(
now
-
create_date
)
/
86400
# Age in days
if
age
>
max_age
[
"data"
]:
try
:
if
os
.
path
.
isdir
(
file
):
print
(
"Removing folder %s..."
%
file
)
if
not
fake
:
shutil
.
rmtree
(
file
)
else
:
print
(
"Removing file %s..."
%
file
)
if
not
fake
:
os
.
remove
(
file
)
except
OSError
:
print
(
traceback
.
print_exc
())
elif
os
.
path
.
isdir
(
file
):
query_name_file
=
os
.
path
.
join
(
file
,
".query"
)
if
os
.
path
.
exists
(
query_name_file
):
with
open
(
query_name_file
)
as
query_file
:
sorted_file
=
Functions
.
get_fasta_file
(
file
,
"query"
,
True
)
if
not
sorted_file
.
endswith
(
".sorted"
):
sorted_file
=
None
if
sorted_file
is
not
None
:
create_date
=
os
.
path
.
getctime
(
sorted_file
)
age
=
(
now
-
create_date
)
/
86400
# Age in days
if
age
>
max_age
[
"fasta_sorted"
]:
print
(
"Removing fasta file %s..."
%
sorted_file
)
if
not
fake
:
os
.
remove
(
sorted_file
)
if
__name__
==
'__main__'
:
...
...
@@ -122,7 +128,7 @@ if __name__ == '__main__':
print
(
"# Parsing Jobs in DB #"
)
print
(
"######################"
)
print
(
""
)
parse_database
(
gallery_jobs
=
parse_database
(
app_data
=
app_data
,
max_age
=
max_age
,
fake
=
fake
...
...
@@ -137,6 +143,7 @@ if __name__ == '__main__':
app_data
=
app_data
,
now
=
now
,
max_age
=
max_age
,
fake
=
fake
fake
=
fake
,
gallery_jobs
=
gallery_jobs
)
print
(
""
)
src/dgenies/database.py
View file @
6c6de5f1
import
os
from
dgenies.config_reader
import
AppConfigReader
from
peewee
import
SqliteDatabase
,
Model
,
CharField
,
IntegerField
,
DateTimeField
,
BooleanField
,
MySQLDatabase
,
OperationalError
from
peewee
import
SqliteDatabase
,
Model
,
CharField
,
IntegerField
,
DateTimeField
,
BooleanField
,
MySQLDatabase
,
\
OperationalError
,
ForeignKeyField
from
playhouse.shortcuts
import
RetryOperationalError
from
datetime
import
datetime
...
...
@@ -64,6 +65,12 @@ class Job(BaseModel):
time_elapsed
=
IntegerField
(
null
=
True
)
class
Gallery
(
BaseModel
):
job
=
ForeignKeyField
(
Job
)
name
=
CharField
()
picture
=
CharField
()
class
Session
(
BaseModel
):
s_id
=
CharField
(
max_length
=
20
,
unique
=
True
)
date_created
=
DateTimeField
()
...
...
@@ -113,5 +120,8 @@ class Session(BaseModel):
if
not
Job
.
table_exists
():
Job
.
create_table
()
if
not
Gallery
.
table_exists
():
Gallery
.
create_table
()
if
not
Session
.
table_exists
():
Session
.
create_table
()
src/dgenies/lib/functions.py
View file @
6c6de5f1
...
...
@@ -10,7 +10,7 @@ from collections import OrderedDict
from
Bio
import
SeqIO
from
jinja2
import
Template
from
dgenies.config_reader
import
AppConfigReader
from
dgenies.database
import
Job
from
dgenies.database
import
Job
,
Gallery
ALLOWED_EXTENSIONS
=
[
'fa'
,
'fasta'
,
'fna'
,
'fa.gz'
,
'fasta.gz'
,
'fna.gz'
]
...
...
@@ -190,3 +190,15 @@ class Functions:
os
.
remove
(
lock_file
)
index
,
sample_name
=
Functions
.
read_index
(
index_file
)
Functions
.
send_fasta_ready
(
mailer
,
job_name
,
sample_name
,
compressed
)
@
staticmethod
def
get_gallery_items
():
items
=
[]
for
item
in
Gallery
.
select
():
items
.
append
({
"name"
:
item
.
name
,
"id_job"
:
item
.
job
.
id_job
,
"picture"
:
item
.
picture
})
return
items
src/dgenies/templates/base.html
View file @
6c6de5f1
...
...
@@ -59,6 +59,7 @@
{% endblock %}
</ul>
</li>
<li
class=
"{% if(menu == 'gallery') %}active{% endif %}"
><a
href=
"/gallery"
>
Gallery
</a></li>
<li
class=
"dropdown"
>
<a
href=
"#"
class=
"dropdown-toggle"
data-toggle=
"dropdown"
>
Documentation
<span
class=
"caret"
></span></a>
...
...
src/dgenies/templates/gallery.html
0 → 100644
View file @
6c6de5f1
{% extends 'base.html' %}
{% block scripts %}
<style>
.item-gallery
{
display
:
inline-block
;
width
:
350px
;
margin-right
:
15px
;
margin-bottom
:
15px
;
margin-top
:
15px
;
}
.item-gallery
p
{
text-align
:
center
;
}
a
:hover
{
text-decoration
:
none
!important
;
}
p
.empty
{
margin-top
:
15px
;
}
</style>
{% endblock %}
{% block content %}
{% if items|length > 0 %}
{% for item in items %}
<a
href=
"/result/{{ item.id_job }}"
>
<div
class=
"item-gallery"
>
<p>
{{ item.name }}
</p>
<img
src=
"/gallery/{{ item.picture }}"
alt=
"illustration"
width=
"100%"
/>
</div>
</a>
{% endfor %}
{% else %}
<p
class=
"empty"
>
Gallery is empty!
</p>
{% endif %}
{% endblock %}
\ No newline at end of file
src/dgenies/views.py
View file @
6c6de5f1
...
...
@@ -6,7 +6,7 @@ import datetime
import
shutil
import
re
import
threading
from
flask
import
render_template
,
request
,
url_for
,
jsonify
,
Response
,
abort
from
flask
import
render_template
,
request
,
url_for
,
jsonify
,
Response
,
abort
,
send_file
from
pathlib
import
Path
from
dgenies.lib.paf
import
Paf
from
dgenies.lib.job_manager
import
JobManager
...
...
@@ -168,6 +168,19 @@ def result(id_res):
return
response
@
app
.
route
(
"/gallery"
,
methods
=
[
'GET'
])
def
gallery
():
return
render_template
(
"gallery.html"
,
items
=
Functions
.
get_gallery_items
(),
menu
=
"gallery"
)
@
app
.
route
(
"/gallery/<filename>"
,
methods
=
[
'GET'
])
def
gallery_file
(
filename
):
try
:
return
send_file
(
os
.
path
.
join
(
config_reader
.
app_data
,
"gallery"
,
filename
))
except
FileNotFoundError
:
abort
(
404
)
def
get_file
(
file
,
gzip
=
False
):
# pragma: no cover
try
:
# Figure out how flask returns static files
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment