Commit 3536adc2 authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Add standalone mode, Implements #114

parent 6ce3b7f3
......@@ -6,11 +6,8 @@ import webbrowser
import threading
from glob import glob
import time
from dgenies.lib.crons import Crons
from dgenies.config_reader import AppConfigReader
from dgenies.bin.clean_jobs import parse_data_folders, parse_database, parse_upload_folders
from dgenies.database import Gallery, Job
from peewee import DoesNotExist
runned = False
......@@ -45,6 +42,8 @@ def parse_args():
default=False)
clear.add_argument("-m", "--max-age", help="Max age for job to delete (0 for all)", type=int, required=False,
default=0)
clear.add_argument("-w", "--web", help="Add this option with -j option, if you use the webserver mode", type=bool,
const=True, nargs="?", required=False, default=False)
# Gallery:
gallery = subparsers.add_parser("gallery", help="Manage gallery")
......@@ -103,11 +102,12 @@ def run(mode="standalone", debug=False, host="127.0.0.1", port=5000, no_crons=Fa
if debug:
os.environ['LOGS'] = "True"
from dgenies import launch
app = launch()
app = launch(mode=mode, debug=debug)
app.run(host=host, port=port, debug=debug)
def clear_crons():
from dgenies.lib.crons import Crons
crons = Crons(None, True)
crons.clear()
......@@ -122,8 +122,7 @@ def clear_logs():
print("No log dir defined!")
def clear_jobs(max_data_age=7):
def clear_jobs(max_data_age=7, web=False):
upload_folder = config.upload_folder
app_data = config.app_data
now = time.time()
......@@ -146,6 +145,7 @@ def clear_jobs(max_data_age=7):
)
print("")
if web:
print("######################")
print("# Parsing Jobs in DB #")
print("######################")
......@@ -155,6 +155,8 @@ def clear_jobs(max_data_age=7):
max_age=max_age
)
print("")
else:
gallery_jobs = []
print("#######################")
print("# Parsing Data folder #")
......@@ -170,6 +172,8 @@ def clear_jobs(max_data_age=7):
def add_to_gallery(id_job, name, picture, query, target):
from dgenies.database import Gallery, Job
from peewee import DoesNotExist
try:
job = Job.get(id_job=id_job)
except DoesNotExist:
......@@ -185,6 +189,7 @@ def add_to_gallery(id_job, name, picture, query, target):
def del_from_gallery_by_id(id_job):
from dgenies.database import Gallery, Job
items = Gallery.select().join(Job).where(Job.id_job == id_job)
list_pictures = []
for item in items:
......@@ -194,6 +199,7 @@ def del_from_gallery_by_id(id_job):
def del_from_gallery_by_name(name):
from dgenies.database import Gallery
items = Gallery.select().where(Gallery.name == name)
list_pictures = []
for item in items:
......@@ -215,7 +221,7 @@ if __name__ == "__main__":
clear_logs()
if args.jobs:
print("Cleaning jobs...")
clear_jobs(args.max_age)
clear_jobs(args.max_age, args.web)
elif command == "gallery_add":
add_to_gallery(args.id_job, args.name, args.pict, args.query, args.target)
elif command == "gallery_del":
......
......@@ -3,7 +3,6 @@
import os
from flask import Flask
from .config_reader import AppConfigReader
from .lib.mailer import Mailer
from .lib.crons import Crons
app = None
......@@ -12,12 +11,16 @@ APP_DATA = None
config_reader = None
mailer = None
app_folder = None
MODE = "webserver"
DEBUG = False
def launch():
global app, app_title, app_folder, APP_DATA, config_reader, mailer
def launch(mode="webserver", debug=False):
global app, app_title, app_folder, APP_DATA, config_reader, mailer, MODE, DEBUG
app_folder = os.path.dirname(os.path.realpath(__file__))
MODE = mode
DEBUG = debug
# Init config reader:
config_reader = AppConfigReader()
......@@ -34,13 +37,15 @@ def launch():
app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
# Init mail:
if MODE == "webserver":
from .lib.mailer import Mailer
mailer = Mailer(app)
if config_reader.debug and config_reader.log_dir != "stdout" and not os.path.exists(config_reader.log_dir):
os.makedirs(config_reader.log_dir)
# Crons:
if os.getenv('DISABLE_CRONS') != "True":
if os.getenv('DISABLE_CRONS') != "True" and MODE == "webserver":
print("Starting crons...")
crons = Crons(app_folder, config_reader.debug or os.getenv('LOGS') == "True")
crons.start_all()
......
......@@ -10,7 +10,6 @@ import argparse
from dgenies.config_reader import AppConfigReader
from dgenies.lib.functions import Functions
from dgenies.database import Job, Gallery
config_reader = AppConfigReader()
......@@ -35,6 +34,7 @@ def parse_upload_folders(upload_folder, now, max_age, fake=False):
def parse_database(app_data, max_age, fake=False):
from dgenies.database import Job, Gallery
gallery_jobs = []
with Job.connect():
old_jobs = Job.select().where(
......
......@@ -6,23 +6,28 @@ from collections import OrderedDict
class Merger:
def __init__(self, paf_in, paf_out, query_in, query_out):
def __init__(self, paf_in, paf_out, query_in, query_out, debug=False):
self.paf_in = paf_in
self.paf_out = paf_out
self.query_in = query_in
self.query_out = query_out
self.debug = debug
def _printer(self, message):
if self.debug:
print(message)
def merge(self):
print("Loading query index...")
self._printer("Loading query index...")
contigs, contigs_split, q_name = self.load_query_index(self.query_in)
print("Merging contigs in PAF file...")
self._printer("Merging contigs in PAF file...")
self.merge_paf(self.paf_in, self.paf_out, contigs, contigs_split)
print("Writing new query index...")
self._printer("Writing new query index...")
self.write_query_index(self.query_out, contigs, q_name)
print("DONE!")
self._printer("DONE!")
@staticmethod
def _get_sorted_splits(contigs_split: dict, all_contigs: dict):
......
......@@ -10,7 +10,7 @@ from collections import OrderedDict
class Splitter:
def __init__(self, input_f, name_f, output_f, size_c=10000000, query_index="query_split.idx"):
def __init__(self, input_f, name_f, output_f, size_c=10000000, query_index="query_split.idx", debug=False):
self.input_f = input_f
self.name_f = name_f
self.size_c = size_c
......@@ -20,6 +20,7 @@ class Splitter:
self.out_dir = os.path.dirname(output_f)
self.index_file = os.path.join(self.out_dir, query_index)
self.nb_contigs = 0
self.debug = debug
def split(self):
"""
......@@ -48,6 +49,7 @@ class Splitter:
return False
chr_name = re.split("\s", line[1:])[0]
fasta_str = ""
if self.debug:
print("Parsing contig \"%s\"... " % chr_name, end="")
elif len(line) > 0:
if next_header or re.match(r"^[ATGCKMRYSWBVHDXN.\-]+$", line.upper()) is None:
......@@ -95,6 +97,7 @@ class Splitter:
self.write_contig(name, seq, enc)
index_f.write("%s\t%d\n" % (name, len(seq)))
nb_contigs = len(contigs)
if self.debug:
if nb_contigs == 1:
print("Keeped!")
else:
......
from dgenies import MODE
import os
from dgenies.config_reader import AppConfigReader
from peewee import SqliteDatabase, Model, CharField, IntegerField, DateTimeField, BooleanField, MySQLDatabase, \
OperationalError, ForeignKeyField
from playhouse.shortcuts import RetryOperationalError
from datetime import datetime
config = AppConfigReader()
db_url = config.database_url
db_type = config.database_type
if MODE == "webserver":
from peewee import SqliteDatabase, Model, CharField, IntegerField, DateTimeField, BooleanField, MySQLDatabase, \
OperationalError, ForeignKeyField
from playhouse.shortcuts import RetryOperationalError
db_url = config.database_url
db_type = config.database_type
class MyRetryDB(RetryOperationalError, MySQLDatabase):
class MyRetryDB(RetryOperationalError, MySQLDatabase):
pass
if db_type == "sqlite":
if db_type == "sqlite":
db = SqliteDatabase(db_url)
elif db_type == "mysql":
elif db_type == "mysql":
db = MyRetryDB(host=config.database_url, port=config.database_port, user=config.database_user,
passwd=config.database_password, database=config.database_db)
else:
else:
raise Exception("Unsupported database type: " + db_type)
class Database:
class Database:
nb_open = 0
......@@ -43,7 +48,7 @@ class Database:
db.close()
class BaseModel(Model):
class BaseModel(Model):
class Meta:
database = db
......@@ -53,7 +58,7 @@ class BaseModel(Model):
return Database()
class Job(BaseModel):
class Job(BaseModel):
id_job = CharField(max_length=50, unique=True)
email = CharField()
id_process = IntegerField(null=True)
......@@ -65,7 +70,7 @@ class Job(BaseModel):
time_elapsed = IntegerField(null=True)
class Gallery(BaseModel):
class Gallery(BaseModel):
job = ForeignKeyField(Job)
name = CharField()
query = CharField()
......@@ -73,7 +78,7 @@ class Gallery(BaseModel):
picture = CharField()
class Session(BaseModel):
class Session(BaseModel):
s_id = CharField(max_length=20, unique=True)
date_created = DateTimeField()
upload_folder = CharField(max_length=20)
......@@ -119,11 +124,34 @@ class Session(BaseModel):
self.save()
if not Job.table_exists():
if not Job.table_exists():
Job.create_table()
if not Gallery.table_exists():
if not Gallery.table_exists():
Gallery.create_table()
if not Session.table_exists():
if not Session.table_exists():
Session.create_table()
else:
class Database:
nb_open = 0
def __init__(self):
pass
def __enter__(self):
pass
def __exit__(self, exc_type, exc_val, exc_tb):
pass
class Job:
@classmethod
def connect(cls):
return Database()
\ No newline at end of file
......@@ -10,7 +10,6 @@ from collections import OrderedDict
from Bio import SeqIO
from jinja2 import Template
from dgenies.config_reader import AppConfigReader
from dgenies.database import Job, Gallery
ALLOWED_EXTENSIONS = ['fa', 'fasta', 'fna', 'fa.gz', 'fasta.gz', 'fna.gz']
......@@ -137,6 +136,7 @@ class Functions:
@staticmethod
def get_mail_for_job(id_job):
from dgenies.database import Job
with Job.connect():
j1 = Job.get(Job.id_job == id_job)
return j1.email
......@@ -221,6 +221,7 @@ class Functions:
@staticmethod
def get_gallery_items():
from dgenies.database import Gallery
items = []
for item in Gallery.select():
items.append({
......
This diff is collapsed.
......@@ -9,15 +9,17 @@ dgenies.run.allowed_ext = [];
dgenies.run.max_upload_file_size = -1
dgenies.run.files = [undefined, undefined];
dgenies.run.allow_upload = false;
dgenies.run.ping_interval = null
dgenies.run.ping_interval = null;
dgenies.run.mode = "webserver"
dgenies.run.init = function (s_id, allowed_ext, max_upload_file_size=1073741824) {
dgenies.run.init = function (s_id, allowed_ext, max_upload_file_size=1073741824, mode="webserver") {
dgenies.run.s_id = s_id;
dgenies.run.allowed_ext = allowed_ext;
dgenies.run.max_upload_file_size = max_upload_file_size
dgenies.run.restore_form();
dgenies.run.set_events();
dgenies.run.init_fileuploads();
dgenies.run.mode = mode;
};
dgenies.run.restore_form = function () {
......@@ -255,7 +257,7 @@ dgenies.run.do_submit = function () {
dgenies.post("/launch_analysis",
{
"id_job": $("input#id_job").val(),
"email": $("input#email").val(),
"email": dgenies.run.mode === "webserver" ? $("input#email").val() : "",
"query": $("input#query").val(),
"query_type": $("select.query").find(":selected").text().toLowerCase(),
"target": $("input#target").val(),
......@@ -300,6 +302,7 @@ dgenies.run.valid_form = function () {
}
// Check mail:
if (dgenies.run.mode === "webserver") {
let email = $("input#email").val();
let mail_re = /^.+@.+\..+$/;
if (email.match(mail_re) === null) {
......@@ -310,8 +313,9 @@ dgenies.run.valid_form = function () {
dgenies.run.add_error("Email is not correct!");
has_errors = true;
}
}
//Check input query:
//Check input target:
if ($("input#target").val().length === 0) {
$("label.file-target").addClass("error");
dgenies.run.add_error("Target fasta is required!");
......
......@@ -59,7 +59,9 @@
{% endblock %}
</ul>
</li>
{% if mode == "webserver" %}
<li class="{% if(menu == 'gallery') %}active{% endif %}"><a href="/gallery">Gallery</a></li>
{% endif %}
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation<span
class="caret"></span></a>
......
......@@ -15,6 +15,11 @@
<h1 class="page-header">D-Genies <small>Summary</small></h1>
<p>Dot plots are widely used to quickly compare sequence sets. They provide a synthetic similarity overview, highlighting repetitions, breaks and inversions. Different tools have been developed to easily generated genomic alignment dot plots, but they are often limited in the input sequence size. D-GENIES is a standalone and WEB application performing large genome alignments using minimap2 software package and generating interactive dot plots. It enables users to sort query sequences along the reference , zoom in the plot and download several image, alignment or sequence files. D-GENIES is an easy to install open source software package (GPL) developed in Python and JavaScript.</p>
{% if pict is none %}
</div>
<div class="col-lg-6" style="text-align:justify">
{% endif %}
<h1 class="page-header">D-Genies <small>Availability</small></h1>
<p>D-Genies is available under the GNU General
Public License (GPL). The package is coming with full documentation and test data. Sources are available <a target="_blank" href="#">here</a>.</p>
......
......@@ -9,7 +9,7 @@
<script src="{{ url_for('static', filename='js/jquery.fileupload-validate.js') }}"></script>
<script src="{{ url_for('static', filename='js/dgenies.run.js') }}" type="text/JavaScript"></script>
{% endblock %}
{% block onload %}dgenies.run.init('{{ s_id }}',{{ allowed_ext }}, {{ max_upload_file_size }});{% endblock %}
{% block onload %}dgenies.run.init('{{ s_id }}',{{ allowed_ext }}, {{ max_upload_file_size }}, '{{ mode }}');{% endblock %}
{% block content %}
<form id="submit_minimap" method=post action="#">
<h2 class="title-launch">Launch map analysis</h2>
......@@ -37,6 +37,7 @@
<input type="text" id="id_job" name="id_job" value="{{ id_job }}" autocomplete="Off" required/>
</td>
</tr>
{% if mode == "webserver" %}
<tr class="email">
<td>
<label for="email" class="email">E-mail*</label>
......@@ -45,6 +46,7 @@
<input type="email" name="email" id="email" value="{{ email }}" required/>
</td>
</tr>
{% endif %}
<tr class="target">
<td>
<label for="file_target" class="file-target">Target fasta**</label>
......
......@@ -54,7 +54,9 @@
{% elif status == "success" %}
<p>Your job was completed successfully.<br/>
Please <a href="/result/{{ id_job }}">click here</a> to show results.</p>
{% if time_elapsed is not none %}
<p>Time elapsed: {{ time_elapsed }}</p>
{% endif %}
{% if target_filtered %}
<p>Note: target fasta has been filtered because it contains too small contigs.<br/>
To see which contigs has been removed from the analysis,
......
from dgenies import app, app_title, app_folder, config_reader, mailer, APP_DATA
from dgenies import app, app_title, app_folder, config_reader, mailer, APP_DATA, MODE
import os
import time
......@@ -13,10 +13,11 @@ from dgenies.lib.job_manager import JobManager
from dgenies.lib.functions import Functions, ALLOWED_EXTENSIONS
from dgenies.lib.upload_file import UploadFile
from dgenies.lib.fasta import Fasta
from dgenies.database import Session, Gallery
from peewee import DoesNotExist
from markdown import Markdown
from markdown.extensions.toc import TocExtension
if MODE == "webserver":
from dgenies.database import Session, Gallery
from peewee import DoesNotExist
@app.context_processor
......@@ -28,18 +29,30 @@ def get_launched_results():
# Main
@app.route("/", methods=['GET'])
def main():
if MODE == "webserver":
pict = Gallery.select().order_by("id")
if len(pict) > 0:
pict = pict[0].picture
else:
pict = None
return render_template("index.html", title=app_title, menu="index", pict=pict)
else:
pict = None
return render_template("index.html", title=app_title, menu="index", pict=pict, mode=MODE)
@app.route("/run", methods=['GET'])
def run():
if MODE == "webserver":
with Session.connect():
s_id = Session.new()
else:
upload_folder = Functions.random_string(20)
tmp_dir = config_reader.upload_folder
upload_folder_path = os.path.join(tmp_dir, upload_folder)
while os.path.exists(upload_folder_path):
upload_folder = Functions.random_string(20)
upload_folder_path = os.path.join(tmp_dir, upload_folder)
s_id = upload_folder
id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
if "id_job" in request.args:
id_job = request.args["id_job"]
......@@ -48,30 +61,35 @@ def run():
email = request.args["email"]
return render_template("run.html", title=app_title, id_job=id_job, email=email,
menu="run", allowed_ext=ALLOWED_EXTENSIONS, s_id=s_id,
max_upload_file_size=config_reader.max_upload_file_size)
max_upload_file_size=config_reader.max_upload_file_size, mode=MODE)
@app.route("/run-test", methods=['GET'])
def run_test():
if MODE == "webserver":
print(config_reader.allowed_ip_tests)
if request.remote_addr not in config_reader.allowed_ip_tests:
return abort(404)
with Session.connect():
return Session.new()
return abort(500)
# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
if MODE == "webserver":
try:
with Session.connect():
session = Session.get(s_id=request.form["s_id"])
except DoesNotExist:
return jsonify({"success": False, "errors": ["Session has expired. Please refresh the page and try again"]})
# Reset session upload:
session.allow_upload = False
session.position = -1
session.save()
upload_folder = session.upload_folder
# Delete session:
session.delete_instance()
else:
upload_folder = request.form["s_id"]
id_job = request.form["id_job"]
email = request.form["email"]
file_query = request.form["query"]
......@@ -86,7 +104,7 @@ def launch_analysis():
errors.append("Id of job not given")
form_pass = False
if email == "":
if email == "" and MODE == "webserver":
errors.append("Email not given")
form_pass = False
if file_target == "":
......@@ -108,7 +126,6 @@ def launch_analysis():
# Save files:
query = None
upload_folder = session.upload_folder
if file_query != "":
query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
query_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_query) \
......@@ -121,10 +138,10 @@ def launch_analysis():
# Launch job:
job = JobManager(id_job, email, query, target, mailer)
if MODE == "webserver":
job.launch()
# Delete session:
session.delete_instance()
else:
job.launch_standalone()
return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
else:
return jsonify({"success": False, "errors": errors})
......@@ -159,13 +176,14 @@ def status(id_job):
error=j_status["error"].replace("#ID#", ""),
id_job=id_job, menu="results", mem_peak=mem_peak,
time_elapsed=time_e,
query_filtered=job.is_query_filtered(), target_filtered=job.is_target_filtered())
query_filtered=job.is_query_filtered(), target_filtered=job.is_target_filtered(), mode=MODE)
# Results path
@app.route("/result/<id_res>", methods=['GET'])
def result(id_res):
my_render = render_template("result.html", title=app_title, id=id_res, menu="result", current_result=id_res)
my_render = render_template("result.html", title=app_title, id=id_res, menu="result", current_result=id_res,
mode=MODE)
response = app.make_response(my_render)
cookie = request.cookies.get("results")
cookie = cookie.split("|") if cookie is not None else []
......@@ -177,15 +195,20 @@ def result(id_res):
@app.route("/gallery", methods=['GET'])
def gallery():
return render_template("gallery.html", items=Functions.get_gallery_items(), menu="gallery", title=app_title)
if MODE == "webserver":
return render_template("gallery.html", items=Functions.get_gallery_items(), menu="gallery", title=app_title,
mode=MODE)
return abort(404)