Commit 89551fe5 authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Add export of query fasta file, Implements #12

parent 135fafd4
......@@ -82,9 +82,11 @@ d3.boxplot.launch = function(res, update=false) {
dgenies.fill_select_zones(res["x_order"], res["y_order"]);
if (res["sorted"]) {
$("input#sort-contigs").val("Undo sort");
$("#export").find("select option[value=4]").show();
}
else {
$("input#sort-contigs").val("Sort contigs");
$("#export").find("select option[value=4]").hide();
}
d3.boxplot.name_x = res["name_x"];
d3.boxplot.name_y = res["name_y"];
......
......@@ -14,6 +14,7 @@ dgenies.result.export.save_file = function(blob, format) {
}
dgenies.result.export.export_png = function() {
dgenies.show_loading("Building files...", 180);
let export_div = $("div#export-pict");
export_div.html("").append($("<canvas>"));
canvg(export_div.find("canvas")[0], dgenies.result.export.get_svg());
......@@ -25,6 +26,7 @@ dgenies.result.export.export_png = function() {
};
dgenies.result.export.export_svg = function () {
dgenies.show_loading("Building files...", 180);
let transform = d3.boxplot.container.attr("transform");
let after = function() {
let blob = new Blob([dgenies.result.export.get_svg()], {type: "image/svg+xml"});
......@@ -36,6 +38,7 @@ dgenies.result.export.export_svg = function () {
};
dgenies.result.export.export_paf = function () {
dgenies.show_loading("Building files...", 180);
let export_div = $("div#export-pict");
export_div.html("");
export_div.append($("<a>").attr("href", `/paf/${dgenies.result.id_res}`)
......@@ -44,21 +47,96 @@ dgenies.result.export.export_paf = function () {
document.getElementById('my-download').click();
};
dgenies.result.export.dl_fasta = function (gzip=false) {
let export_div = $("div#export-pict");
export_div.html("");
export_div.append($("<a>").attr("href", `/fasta-query/${dgenies.result.id_res}`)
.attr("download", d3.boxplot.name_y + (gzip ? ".fasta.gz" : ".fasta")).attr("id", "my-download").text("download"));
dgenies.hide_loading();
document.getElementById('my-download').click();
};
dgenies.result.export.export_fasta = function(compress=false) {
dgenies.show_loading("Building files...", 180);
dgenies.post("/get-fasta-query/" + dgenies.result.id_res,
{
gzip: compress
},
function(data, success) {
console.log(data);
if (data["status"] === 0) {
window.setTimeout(() => {
dgenies.result.export.export_fasta();
}, 10000)
}
else if (data["status"] === 2) {
dgenies.result.export.dl_fasta(data["gzip"]);
}
else if (data["status"] === 1) {
dgenies.hide_loading();
dgenies.notify("We are building your Fasta file. You will receive by mail a link to download it soon!",
"info");
}
else {
dgenies.hide_loading();
dgenies.notify("An error has occurred. Please contact us to report the bug", "danger");
}
});
};
dgenies.result.export.ask_export_fasta = function () {
let dialog = $("<div>")
.attr("id", "dialog-confirm")
.attr("title", "Gzip?");
let icon = $("<span>")
.attr("class", "ui-icon ui-icon-help")
.css("float", "left")
.css("margin", "12px 12px 20px 0");
let body = $("<p>");
body.append(icon);
body.append("Compression is recommanded on slow connections. Download Gzip file?");
dialog.append(body);
dialog.dialog({
resizable: false,
height: "auto",
width: 500,
modal: true,
buttons: {
"Use default": function() {
$( this ).dialog( "close" );
dgenies.result.export.export_fasta(false);
},
"Use Gzip": function () {
$( this ).dialog( "close" );
dgenies.result.export.export_fasta(true);
},
Cancel: function () {
$( this ).dialog( "close" );
}
}
})
};
dgenies.result.export.export = function () {
let select = $("form#export select");
let selection = parseInt(select.val());
dgenies.show_loading("Building files...", 180);
window.setTimeout(() => {
if (selection > 0) {
let async = false;
if (selection === 1)
dgenies.result.export.export_svg();
else if (selection === 2)
dgenies.result.export.export_png();
else if (selection === 3)
dgenies.result.export.export_paf();
else if (selection === 4) {
dgenies.result.export.ask_export_fasta();
async = true;
}
else
dgenies.notify("Not supported yet!", "danger", 2000);
dgenies.hide_loading();
if (!async)
dgenies.hide_loading();
select.val("0");
}
}, 0);
......
......@@ -3,8 +3,16 @@ import random
import string
import gzip
import io
import shutil
import re
import traceback
from lib.Fasta import Fasta
from collections import OrderedDict
from Bio import SeqIO
from jinja2 import Template
from config_reader import AppConfigReader
from database import Job
from pony.orm import db_session
ALLOWED_EXTENSIONS = ['fa', 'fasta', 'fna', 'fa.gz', 'fasta.gz', 'fna.gz']
......@@ -56,4 +64,141 @@ class Functions:
elif len(line) > 0:
len_c += len(line)
if contig is not None and len_c > 0:
out_file.write("%s\t%d\n" % (contig, len_c))
\ No newline at end of file
out_file.write("%s\t%d\n" % (contig, len_c))
@staticmethod
def __get_do_sort(fasta, is_sorted):
do_sort = False
if is_sorted:
do_sort = True
if fasta.endswith(".sorted"):
do_sort = False
return do_sort
@staticmethod
def get_fasta_file(res_dir, type_f, is_sorted):
fasta_file = None
try:
with open(os.path.join(res_dir, "." + type_f), "r") as save_name:
fasta_file = save_name.readline()
except IOError:
print(res_dir + ": Unable to load saved name for " + type_f)
pass
if fasta_file is not None and os.path.exists(fasta_file):
fasta_file_uc = fasta_file
if fasta_file.endswith(".gz"):
fasta_file_uc = fasta_file[:-3]
if is_sorted:
sorted_fasta = fasta_file_uc + ".sorted"
if os.path.exists(sorted_fasta):
fasta_file = sorted_fasta
else:
sorted_fasta = fasta_file_uc + ".gz.sorted"
if os.path.exists(sorted_fasta):
fasta_file = sorted_fasta
return fasta_file
@staticmethod
def uncompress(filename):
try:
uncompressed = filename.rsplit('.', 1)[0]
parts = uncompressed.rsplit("/", 1)
file_path = parts[0]
basename = parts[1]
n = 2
while os.path.exists(uncompressed):
uncompressed = "%s/%d_%s" % (file_path, n, basename)
n += 1
with open(filename, "rb") as infile, open(uncompressed, "wb") as outfile:
outfile.write(gzip.decompress(infile.read()))
return uncompressed
except Exception as e:
print(traceback.format_exc())
return None
@staticmethod
def compress(filename):
try:
if not filename.endswith(".gz") and not filename.endswith(".gz.sorted"):
compressed = filename + ".gz" if not filename.endswith(".sorted") else filename[:-7] + ".gz.sorted"
parts = compressed.rsplit("/", 1)
file_path = parts[0]
basename = parts[1]
n = 2
while os.path.exists(compressed):
compressed = "%s/%d_%s" % (file_path, n, basename)
n += 1
with open(filename, "rb") as infile, gzip.open(compressed, "wb") as outfile:
shutil.copyfileobj(infile, outfile)
os.remove(filename)
return compressed
return filename
except Exception as e:
print(traceback.format_exc())
return None
@staticmethod
def read_index(index_file):
index = OrderedDict()
with open(index_file, "r") as index_f:
lines = index_f.readlines()
for line in lines[1:]:
if line != "":
parts = line.strip("\n").split("\t")
name = parts[0]
lenght = int(parts[1])
to_reverse = parts[2] == "1" if len(parts) >= 3 else False
index[name] = {
"length": lenght,
"to_reverse": to_reverse
}
return index
@staticmethod
@db_session
def get_mail_for_job(id_job):
j1 = Job.get(id_job=id_job)
return j1.email
@staticmethod
def send_fasta_ready(mailer, job_name):
config_reader = AppConfigReader()
web_url = config_reader.get_web_url()
with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "mail_templates", "dl_fasta.html")) \
as t_file:
template = Template(t_file.read())
message_html = template.render(job_name=job_name, status="success", url_base=web_url)
message = "D-Genies\n\n" \
"Job %s - Download fasta\n\n" % job_name
message += "Query fasta file for job %s is ready to download." % job_name
message += "You can click on the link below to download it:\n\n"
message += "%s/fasta-query/%s" % (web_url, job_name)
mailer.send_mail([Functions.get_mail_for_job(job_name)], "Job %s - Download fasta" % job_name, message,
message_html)
@staticmethod
def sort_fasta(job_name, fasta_file, index_file, lock_file, compress=False, mailer=None):
print("Loading index...")
index = Functions.read_index(index_file)
print("Starting fasta sort...")
is_compressed = fasta_file.endswith(".gz")
if is_compressed:
fasta_file = Functions.uncompress(fasta_file)
seq = SeqIO.index(fasta_file, "fasta")
fasta_file_o = fasta_file + ".sorted"
with open(fasta_file_o, "w") as fasta_out:
for name, props in index.items():
sequence = seq[name]
if props["to_reverse"]:
sequence = sequence[::-1]
SeqIO.write(sequence, fasta_out, "fasta")
if is_compressed:
os.remove(fasta_file)
if compress:
print("Compress...")
Functions.compress(fasta_file_o)
os.remove(lock_file)
if mailer is not None:
Functions.send_fasta_ready(mailer, job_name)
print("Fasta sort done!")
......@@ -100,14 +100,22 @@ class JobManager:
db.commit()
return status == "success"
def __getting_local_file(self, fasta: Fasta):
finale_path = os.path.join(self.output_dir, os.path.basename(fasta.get_path()))
def __getting_local_file(self, fasta: Fasta, type_f):
finale_path = os.path.join(self.output_dir, type_f + "_" + os.path.basename(fasta.get_path()))
shutil.move(fasta.get_path(), finale_path)
with open(os.path.join(self.output_dir, "." + type_f), "w") as save_file:
save_file.write(finale_path)
return finale_path
def __getting_file_from_url(self, fasta: Fasta):
finale_path = wget.download(fasta.get_path(), self.output_dir, None)
return finale_path
def __getting_file_from_url(self, fasta: Fasta, type_f):
dl_path = wget.download(fasta.get_path(), self.output_dir, None)
filename = os.path.basename(dl_path)
name = os.path.splitext(filename.replace(".gz", ""))[0]
finale_path = os.path.join(self.output_dir, type_f + "_" + filename)
shutil.move(dl_path, finale_path)
with open(os.path.join(self.output_dir, "." + type_f), "w") as save_file:
save_file.write(finale_path)
return finale_path, name
@db_session
def __check_url(self, fasta: Fasta):
......@@ -143,20 +151,18 @@ class JobManager:
correct = True
if self.query is not None:
if self.query.get_type() == "local":
self.query.set_path(self.__getting_local_file(self.query))
self.query.set_path(self.__getting_local_file(self.query, "query"))
elif self.__check_url(self.query):
finale_path = self.__getting_file_from_url(self.query)
filename = os.path.splitext(os.path.basename(finale_path).replace(".gz", ""))[0]
finale_path, filename = self.__getting_file_from_url(self.query)
self.query.set_path(finale_path)
self.query.set_name(filename)
else:
correct = False
if correct and self.target is not None:
if self.target.get_type() == "local":
self.target.set_path(self.__getting_local_file(self.target))
self.target.set_path(self.__getting_local_file(self.target, "target"))
elif self.__check_url(self.target):
finale_path = self.__getting_file_from_url(self.target)
filename = os.path.splitext(os.path.basename(finale_path).replace(".gz", ""))[0]
finale_path, filename = self.__getting_file_from_url(self.target)
self.target.set_path(finale_path)
self.target.set_name(filename)
else:
......
<style>
.inline {
display: inline-block;
vertical-align: middle;
}
.header {
background: #21264a;
padding: 5px;
}
.header h1 {
margin: 0;
color: white;
}
</style>
<div class="header">
<img src="{{ url_base }}/static/images/logo.png" height="45px" alt="" class="inline" style="margin-right: 5px;"/>
<h1 class="inline">D-Genies</h1>
</div>
<h3>
{% if status == "success" %}
Job {{ job_name }} - Download fasta
{% else %}
Job {{ job_name }} - Build of fasta failed
{% endif %}
</h3>
<p>Hi,</p>
{% if status == "success" %}
<p>Query fasta file for job {{ job_name }} is ready to download.
You can <a href="{{ url_base }}/fasta-query/{{ job_name }}">click here</a> to download it.</p>
{% else %}
<p>Build of query fasta file for job {{ job_name }} has failed. You can try again. If the problem persists, please contact the support.</p>
{% endif %}
<p>See you soon on D-Genies,</p>
<p>The D-Genies team</p>
\ No newline at end of file
......@@ -11,18 +11,34 @@ class Mailer:
self.mail_status = config_reader.get_mail_status_sender()
self.mail_reply = config_reader.get_mail_reply()
self.mail_org = config_reader.get_mail_org()
self.disable = config_reader.get_disable_mail()
def __send_async_email(self, msg):
with self.app.app_context():
self.mail.send(msg)
def send_mail(self, recipients: list, subject: str, message: str, message_html: str=None):
msg = Message(
subject= subject,
recipients=recipients,
html=message_html,
body=message,
sender=(self.mail_org, self.mail_status) if self.mail_org is not None else self.mail_status,
reply_to=self.mail_reply
)
self.__send_async_email(msg)
sender = (self.mail_org, self.mail_status) if self.mail_org is not None else self.mail_status
reply = self.mail_reply
if not self.disable:
msg = Message(
subject= subject,
recipients=recipients,
html=message_html,
body=message,
sender=sender,
reply_to=reply
)
self.__send_async_email(msg)
else: # Print debug
print("################\n"
"# WARNING !!!! #\n"
"################\n\n"
"!!! SEND MAILS DISABLED BY CONFIGURATION !!!\n\n"
"(This might be disabled in production)\n\n")
print("Sender: %s <%s>\n" % sender)
print("Reply to: %s\n" % reply)
print("Recipients: %s\n" % ", ".join(recipients))
print("Subject: %s\n" % subject)
print("Message:\n%s\n\n" % message)
print("Message HTML:\n%s\n\n" % message_html)
......@@ -3,6 +3,7 @@
import os
from math import sqrt
from numpy import mean
from pathlib import Path
class Paf:
......@@ -14,7 +15,7 @@ class Paf:
self.idx_q = idx_q
self.idx_t = idx_t
self.sorted = False
if os.path.exists(paf + ".sorted") and os.path.exists(idx_q + ".sorted"):
if os.path.exists(os.path.join(os.path.dirname(paf), ".sorted")):
self.paf += ".sorted"
self.idx_q += ".sorted"
self.sorted = True
......@@ -273,6 +274,15 @@ class Paf:
idx.write("\t".join([contig, str(self.q_contigs[contig]), "1" if contig in contigs_reoriented else "0"])
+ "\n")
def set_sorted(self, is_sorted):
self.sorted = is_sorted
sorted_touch = os.path.join(os.path.dirname(self.paf), ".sorted")
if is_sorted:
Path(sorted_touch).touch()
else:
if os.path.exists(sorted_touch):
os.remove(sorted_touch)
def sort(self):
self.parse_paf(False)
if not self.sorted: # Do the sort
......@@ -351,7 +361,7 @@ class Paf:
# Update index:
self._update_query_index(reorient_contigs)
self.sorted = True
self.set_sorted(True)
else: # Undo the sort
if os.path.exists(self.paf):
......@@ -360,7 +370,7 @@ class Paf:
if os.path.exists(self.idx_q):
os.remove(self.idx_q)
self.idx_q = self.idx_q.replace(".sorted", "")
self.sorted = False
self.set_sorted(False)
# Re parse PAF file:
self.parsed = False
......
......@@ -6,3 +6,4 @@ docopt==0.6.*
numpy
wget==3.2
requests==2.18.*
biopython==1.70
......@@ -87,6 +87,12 @@ class AppConfigReader(object):
def get_send_mail_status(self):
try:
return self.replace_vars(self.reader.get("mail", "send_mail_status")).lower() == "true"
return self.reader.get("mail", "send_mail_status").lower() == "true"
except NoOptionError:
return True
def get_disable_mail(self):
try:
return self.reader.get("mail", "disable").lower() == "true"
except NoOptionError:
return False
......@@ -5,8 +5,9 @@ import time
import datetime
import shutil
import re
import threading
from flask import Flask, render_template, request, url_for, jsonify, session, Response, abort
from flask_mail import Mail
from pathlib import Path
from lib.paf import Paf
from config_reader import AppConfigReader
from lib.job_manager import JobManager
......@@ -147,14 +148,14 @@ def result(id_res):
return response
def get_file(file): # pragma: no cover
def get_file(file, gzip=False): # pragma: no cover
try:
# Figure out how flask returns static files
# Tried:
# - render_template
# - send_file
# This should not be so non-obvious
return open(file).read()
return open(file, "rb" if gzip else "r").read()
except IOError as exc:
return str(exc)
......@@ -201,6 +202,55 @@ def sort_graph(id_res):
return jsonify({"success": False, "message": paf.error})
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
res_dir = os.path.join(app_data, id_res)
lock_query = os.path.join(res_dir, ".query-fasta-build")
is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
if query_fasta is not None:
if is_sorted and not query_fasta.endswith(".sorted"):
# Do the sort
Path(lock_query).touch()
thread = threading.Timer(1, Functions.sort_fasta, kwargs={
"job_name": id_res,
"fasta_file": query_fasta,
"index_file": os.path.join(res_dir, "query.idx.sorted"),
"lock_file": lock_query,
"compress": request.form["gzip"],
"mailer": mailer
})
thread.start()
return jsonify({"success": True, "status": 0, "status_message": "Started"})
elif is_sorted and os.path.exists(lock_query):
# Sort is already in progress
return jsonify({"success": True, "status": 1, "status_message": "In progress"})
else:
# No sort to do or sort done
return jsonify({"success": True, "status": 2, "status_message": "Done",
"gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
else:
return jsonify({"success": False,
"message": "Unable to get fasta file for query. Please contact us to report the bug"})
@app.route('/fasta-query/<id_res>', methods=['GET'])
def dl_fasta(id_res):
res_dir = os.path.join(app_data, id_res)
lock_query = os.path.join(res_dir, ".query-fasta-build")
is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
if not os.path.exists(lock_query) or not is_sorted:
query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
print(query_fasta)
if query_fasta is not None:
if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
content = get_file(query_fasta, True)
return Response(content, mimetype="application/gzip")
content = get_file(query_fasta)
return Response(content, mimetype="text/plain")
abort(404)
@app.route("/upload", methods=['POST'])
def upload():
if "user_tmp_dir" in session and session["user_tmp_dir"] != "":
......
......@@ -39,7 +39,7 @@
<option value="1">Svg</option>
<option value="2">Png</option>
<option value="3">Paf file</option>
<option value="4">Fasta files</option>
<option value="4">Query Fasta</option>
</select>
</form>
</div>
......