Commit 6c6de5f1 authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Add gallery, Implements #87

parent f2cadc7d
...@@ -9,9 +9,13 @@ import time ...@@ -9,9 +9,13 @@ import time
from dgenies.lib.crons import Crons from dgenies.lib.crons import Crons
from dgenies.config_reader import AppConfigReader from dgenies.config_reader import AppConfigReader
from dgenies.bin.clean_jobs import parse_data_folders, parse_database, parse_upload_folders from dgenies.bin.clean_jobs import parse_data_folders, parse_database, parse_upload_folders
from dgenies.database import Gallery, Job
from peewee import DoesNotExist
runned = False runned = False
config = AppConfigReader()
def parse_args(): def parse_args():
parser = argparse.ArgumentParser(description="Manage dgenies application") parser = argparse.ArgumentParser(description="Manage dgenies application")
...@@ -42,6 +46,28 @@ def parse_args(): ...@@ -42,6 +46,28 @@ def parse_args():
clear.add_argument("-m", "--max-age", help="Max age for job to delete (0 for all)", type=int, required=False, clear.add_argument("-m", "--max-age", help="Max age for job to delete (0 for all)", type=int, required=False,
default=0) default=0)
# Gallery:
gallery = subparsers.add_parser("gallery", help="Manage gallery")
subparsers_gallery = gallery.add_subparsers()
# Gallery add:
gallery_add = subparsers_gallery.add_parser("add", help="Add new job to the gallery")
gallery_add.set_defaults(which="add")
gallery_add.add_argument("-i", "--id-job", type=str, required=True,
help="Id (name) of the job to add to the gallery")
gallery_add.add_argument("-n", "--name", type=str, required=True, help="Name to show in the gallery for the job")
gallery_add.add_argument("-p", "--pict", type=str, required=True, help="Name of the file that illustrate the job")
# Gallery del:
gallery_del = subparsers_gallery.add_parser("del", help="Delete a job from the gallery")
gallery_del.set_defaults(which="del")
gallery_del.add_argument("-i", "--id-job", type=str, required=False,
help="Id (name) of the job to delete from the gallery")
gallery_del.add_argument("-n", "--name", type=str, required=False,
help="Name of the job shown in the gallery")
gallery_del.add_argument("--remove-pict", help="Clear crons", type=bool, const=True, nargs="?", required=False,
default=False)
args = parser.parse_args() args = parser.parse_args()
if args.subparser_name == "run": if args.subparser_name == "run":
...@@ -51,6 +77,8 @@ def parse_args(): ...@@ -51,6 +77,8 @@ def parse_args():
print("Nothing to do.") print("Nothing to do.")
return "clear", None return "clear", None
return "clear", args return "clear", args
if args.subparser_name == "gallery":
return "gallery_" + args.which, args
def start_browser(host, port): def start_browser(host, port):
...@@ -79,8 +107,6 @@ def clear_crons(): ...@@ -79,8 +107,6 @@ def clear_crons():
def clear_logs(): def clear_logs():
config = AppConfigReader()
if hasattr(config, "log_dir"): if hasattr(config, "log_dir"):
log_files = glob(os.path.join(config.log_dir, "*.log")) log_files = glob(os.path.join(config.log_dir, "*.log"))
for file in log_files: for file in log_files:
...@@ -91,10 +117,9 @@ def clear_logs(): ...@@ -91,10 +117,9 @@ def clear_logs():
def clear_jobs(max_data_age=7): def clear_jobs(max_data_age=7):
config_reader = AppConfigReader()
upload_folder = config_reader.upload_folder upload_folder = config.upload_folder
app_data = config_reader.app_data app_data = config.app_data
now = time.time() now = time.time()
max_age = { max_age = {
...@@ -119,7 +144,7 @@ def clear_jobs(max_data_age=7): ...@@ -119,7 +144,7 @@ def clear_jobs(max_data_age=7):
print("# Parsing Jobs in DB #") print("# Parsing Jobs in DB #")
print("######################") print("######################")
print("") print("")
parse_database( gallery_jobs = parse_database(
app_data=app_data, app_data=app_data,
max_age=max_age max_age=max_age
) )
...@@ -132,11 +157,44 @@ def clear_jobs(max_data_age=7): ...@@ -132,11 +157,44 @@ def clear_jobs(max_data_age=7):
parse_data_folders( parse_data_folders(
app_data=app_data, app_data=app_data,
now=now, now=now,
max_age=max_age max_age=max_age,
gallery_jobs=gallery_jobs
) )
print("") print("")
def add_to_gallery(id_job, name, picture):
try:
job = Job.get(id_job=id_job)
except DoesNotExist:
print("Error: job \"%s\" does not exists!" % id_job)
exit(1)
pict_file = os.path.join(config.app_data, "gallery", picture)
if not os.path.exists(pict_file):
print("Error: file \"%s\" does not exists!" % pict_file)
exit(1)
item = Gallery.create(job=job, name=name, picture=picture)
item.save()
def del_from_gallery_by_id(id_job):
items = Gallery.select().join(Job).where(Job.id_job == id_job)
list_pictures = []
for item in items:
list_pictures.append(item.picture)
item.delete_instance()
return list_pictures
def del_from_gallery_by_name(name):
items = Gallery.select().where(Gallery.name == name)
list_pictures = []
for item in items:
list_pictures.append(item.picture)
item.delete_instance()
return list_pictures
if __name__ == "__main__": if __name__ == "__main__":
command, args = parse_args() command, args = parse_args()
if command == "run": if command == "run":
...@@ -151,3 +209,19 @@ if __name__ == "__main__": ...@@ -151,3 +209,19 @@ if __name__ == "__main__":
if args.jobs: if args.jobs:
print("Cleaning jobs...") print("Cleaning jobs...")
clear_jobs(args.max_age) clear_jobs(args.max_age)
elif command == "gallery_add":
add_to_gallery(args.id_job, args.name, args.pict)
elif command == "gallery_del":
if args.id_job is None and args.name is None:
print("Error: please give an id or a name for the job!")
exit(1)
if args.id_job is not None:
pictures = del_from_gallery_by_id(args.id_job)
else:
pictures = del_from_gallery_by_name(args.name)
if args.remove_pict:
for picture in pictures:
try:
os.remove(os.path.join(config.app_data, "gallery", picture))
except FileNotFoundError:
pass
...@@ -10,7 +10,7 @@ import argparse ...@@ -10,7 +10,7 @@ import argparse
from dgenies.config_reader import AppConfigReader from dgenies.config_reader import AppConfigReader
from dgenies.lib.functions import Functions from dgenies.lib.functions import Functions
from dgenies.database import Job from dgenies.database import Job, Gallery
config_reader = AppConfigReader() config_reader = AppConfigReader()
...@@ -35,6 +35,7 @@ def parse_upload_folders(upload_folder, now, max_age, fake=False): ...@@ -35,6 +35,7 @@ def parse_upload_folders(upload_folder, now, max_age, fake=False):
def parse_database(app_data, max_age, fake=False): def parse_database(app_data, max_age, fake=False):
gallery_jobs = []
with Job.connect(): with Job.connect():
old_jobs = Job.select().where( old_jobs = Job.select().where(
((Job.status == "success") & (Job.date_created < datetime.now() - timedelta(days=max_age["data"]))) ((Job.status == "success") & (Job.date_created < datetime.now() - timedelta(days=max_age["data"])))
...@@ -43,6 +44,9 @@ def parse_database(app_data, max_age, fake=False): ...@@ -43,6 +44,9 @@ def parse_database(app_data, max_age, fake=False):
) )
for job in old_jobs: for job in old_jobs:
id_job = job.id_job id_job = job.id_job
is_gallery = len(Gallery.select().join(Job).where(Job.id_job == id_job)) > 0
if is_gallery:
gallery_jobs.append(id_job)
print("Removing job %s..." % id_job) print("Removing job %s..." % id_job)
data_dir = os.path.join(app_data, id_job) data_dir = os.path.join(app_data, id_job)
if os.path.exists(data_dir) and os.path.isdir(data_dir): if os.path.exists(data_dir) and os.path.isdir(data_dir):
...@@ -52,39 +56,41 @@ def parse_database(app_data, max_age, fake=False): ...@@ -52,39 +56,41 @@ def parse_database(app_data, max_age, fake=False):
print("Job %s has no data folder!" % id_job) print("Job %s has no data folder!" % id_job)
if not fake: if not fake:
job.delete_instance() job.delete_instance()
return gallery_jobs
def parse_data_folders(app_data, now, max_age, fake=False): def parse_data_folders(app_data, now, max_age, gallery_jobs, fake=False):
for file in os.listdir(app_data): for file in os.listdir(app_data):
file = os.path.join(app_data, file) if file not in gallery_jobs:
create_date = os.path.getctime(file) file = os.path.join(app_data, file)
age = (now - create_date) / 86400 # Age in days create_date = os.path.getctime(file)
if age > max_age["data"]: age = (now - create_date) / 86400 # Age in days
try: if age > max_age["data"]:
if os.path.isdir(file): try:
print("Removing folder %s..." % file) if os.path.isdir(file):
if not fake: print("Removing folder %s..." % file)
shutil.rmtree(file) if not fake:
else: shutil.rmtree(file)
print("Removing file %s..." % file) else:
if not fake: print("Removing file %s..." % file)
os.remove(file) if not fake:
except OSError: os.remove(file)
print(traceback.print_exc()) except OSError:
elif os.path.isdir(file): print(traceback.print_exc())
query_name_file = os.path.join(file, ".query") elif os.path.isdir(file):
if os.path.exists(query_name_file): query_name_file = os.path.join(file, ".query")
with open(query_name_file) as query_file: if os.path.exists(query_name_file):
sorted_file = Functions.get_fasta_file(file, "query", True) with open(query_name_file) as query_file:
if not sorted_file.endswith(".sorted"): sorted_file = Functions.get_fasta_file(file, "query", True)
sorted_file = None if not sorted_file.endswith(".sorted"):
if sorted_file is not None: sorted_file = None
create_date = os.path.getctime(sorted_file) if sorted_file is not None:
age = (now - create_date) / 86400 # Age in days create_date = os.path.getctime(sorted_file)
if age > max_age["fasta_sorted"]: age = (now - create_date) / 86400 # Age in days
print("Removing fasta file %s..." % sorted_file) if age > max_age["fasta_sorted"]:
if not fake: print("Removing fasta file %s..." % sorted_file)
os.remove(sorted_file) if not fake:
os.remove(sorted_file)
if __name__ == '__main__': if __name__ == '__main__':
...@@ -122,7 +128,7 @@ if __name__ == '__main__': ...@@ -122,7 +128,7 @@ if __name__ == '__main__':
print("# Parsing Jobs in DB #") print("# Parsing Jobs in DB #")
print("######################") print("######################")
print("") print("")
parse_database( gallery_jobs = parse_database(
app_data=app_data, app_data=app_data,
max_age=max_age, max_age=max_age,
fake=fake fake=fake
...@@ -137,6 +143,7 @@ if __name__ == '__main__': ...@@ -137,6 +143,7 @@ if __name__ == '__main__':
app_data=app_data, app_data=app_data,
now=now, now=now,
max_age=max_age, max_age=max_age,
fake=fake fake=fake,
gallery_jobs=gallery_jobs
) )
print("") print("")
import os import os
from dgenies.config_reader import AppConfigReader from dgenies.config_reader import AppConfigReader
from peewee import SqliteDatabase, Model, CharField, IntegerField, DateTimeField, BooleanField, MySQLDatabase, OperationalError from peewee import SqliteDatabase, Model, CharField, IntegerField, DateTimeField, BooleanField, MySQLDatabase, \
OperationalError, ForeignKeyField
from playhouse.shortcuts import RetryOperationalError from playhouse.shortcuts import RetryOperationalError
from datetime import datetime from datetime import datetime
...@@ -64,6 +65,12 @@ class Job(BaseModel): ...@@ -64,6 +65,12 @@ class Job(BaseModel):
time_elapsed = IntegerField(null=True) time_elapsed = IntegerField(null=True)
class Gallery(BaseModel):
job = ForeignKeyField(Job)
name = CharField()
picture = CharField()
class Session(BaseModel): class Session(BaseModel):
s_id = CharField(max_length=20, unique=True) s_id = CharField(max_length=20, unique=True)
date_created = DateTimeField() date_created = DateTimeField()
...@@ -113,5 +120,8 @@ class Session(BaseModel): ...@@ -113,5 +120,8 @@ class Session(BaseModel):
if not Job.table_exists(): if not Job.table_exists():
Job.create_table() Job.create_table()
if not Gallery.table_exists():
Gallery.create_table()
if not Session.table_exists(): if not Session.table_exists():
Session.create_table() Session.create_table()
...@@ -10,7 +10,7 @@ from collections import OrderedDict ...@@ -10,7 +10,7 @@ from collections import OrderedDict
from Bio import SeqIO from Bio import SeqIO
from jinja2 import Template from jinja2 import Template
from dgenies.config_reader import AppConfigReader from dgenies.config_reader import AppConfigReader
from dgenies.database import Job from dgenies.database import Job, Gallery
ALLOWED_EXTENSIONS = ['fa', 'fasta', 'fna', 'fa.gz', 'fasta.gz', 'fna.gz'] ALLOWED_EXTENSIONS = ['fa', 'fasta', 'fna', 'fa.gz', 'fasta.gz', 'fna.gz']
...@@ -190,3 +190,15 @@ class Functions: ...@@ -190,3 +190,15 @@ class Functions:
os.remove(lock_file) os.remove(lock_file)
index, sample_name = Functions.read_index(index_file) index, sample_name = Functions.read_index(index_file)
Functions.send_fasta_ready(mailer, job_name, sample_name, compressed) Functions.send_fasta_ready(mailer, job_name, sample_name, compressed)
@staticmethod
def get_gallery_items():
items = []
for item in Gallery.select():
items.append({
"name": item.name,
"id_job": item.job.id_job,
"picture": item.picture
})
return items
...@@ -59,6 +59,7 @@ ...@@ -59,6 +59,7 @@
{% endblock %} {% endblock %}
</ul> </ul>
</li> </li>
<li class="{% if(menu == 'gallery') %}active{% endif %}"><a href="/gallery">Gallery</a></li>
<li class="dropdown"> <li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation<span <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation<span
class="caret"></span></a> class="caret"></span></a>
......
{% extends 'base.html' %}
{% block scripts %}
<style>
.item-gallery {
display: inline-block;
width: 350px;
margin-right: 15px;
margin-bottom: 15px;
margin-top: 15px;
}
.item-gallery p {
text-align: center;
}
a:hover {
text-decoration: none !important;
}
p.empty {
margin-top: 15px;
}
</style>
{% endblock %}
{% block content %}
{% if items|length > 0 %}
{% for item in items %}
<a href="/result/{{ item.id_job }}">
<div class="item-gallery">
<p>{{ item.name }}</p>
<img src="/gallery/{{ item.picture }}" alt="illustration" width="100%"/>
</div>
</a>
{% endfor %}
{% else %}
<p class="empty">Gallery is empty!</p>
{% endif %}
{% endblock %}
\ No newline at end of file
...@@ -6,7 +6,7 @@ import datetime ...@@ -6,7 +6,7 @@ import datetime
import shutil import shutil
import re import re
import threading import threading
from flask import render_template, request, url_for, jsonify, Response, abort from flask import render_template, request, url_for, jsonify, Response, abort, send_file
from pathlib import Path from pathlib import Path
from dgenies.lib.paf import Paf from dgenies.lib.paf import Paf
from dgenies.lib.job_manager import JobManager from dgenies.lib.job_manager import JobManager
...@@ -168,6 +168,19 @@ def result(id_res): ...@@ -168,6 +168,19 @@ def result(id_res):
return response return response
@app.route("/gallery", methods=['GET'])
def gallery():
return render_template("gallery.html", items=Functions.get_gallery_items(), menu="gallery")
@app.route("/gallery/<filename>", methods=['GET'])
def gallery_file(filename):
try:
return send_file(os.path.join(config_reader.app_data, "gallery", filename))
except FileNotFoundError:
abort(404)
def get_file(file, gzip=False): # pragma: no cover def get_file(file, gzip=False): # pragma: no cover
try: try:
# Figure out how flask returns static files # Figure out how flask returns static files
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment