Add export of query fasta file, Implements #12

js/d3.boxplot.js

@@ -82,9 +82,11 @@ d3.boxplot.launch = function(res, update=false) {
     dgenies.fill_select_zones(res["x_order"], res["y_order"]);
     if (res["sorted"]) {
         $("input#sort-contigs").val("Undo sort");
+        $("#export").find("select option[value=4]").show();
     }
     else {
         $("input#sort-contigs").val("Sort contigs");
+        $("#export").find("select option[value=4]").hide();
     }
     d3.boxplot.name_x = res["name_x"];
     d3.boxplot.name_y = res["name_y"];

js/dgenies.result.export.js

@@ -14,6 +14,7 @@ dgenies.result.export.save_file = function(blob, format) {
 }
 
 dgenies.result.export.export_png = function() {
+    dgenies.show_loading("Building files...", 180);
     let export_div = $("div#export-pict");
     export_div.html("").append($("<canvas>"));
     canvg(export_div.find("canvas")[0], dgenies.result.export.get_svg());
@@ -25,6 +26,7 @@ dgenies.result.export.export_png = function() {
 };
 
 dgenies.result.export.export_svg = function () {
+    dgenies.show_loading("Building files...", 180);
     let transform = d3.boxplot.container.attr("transform");
     let after = function() {
         let blob = new Blob([dgenies.result.export.get_svg()], {type: "image/svg+xml"});
@@ -36,6 +38,7 @@ dgenies.result.export.export_svg = function () {
 };
 
 dgenies.result.export.export_paf = function () {
+    dgenies.show_loading("Building files...", 180);
     let export_div = $("div#export-pict");
     export_div.html("");
     export_div.append($("<a>").attr("href", `/paf/${dgenies.result.id_res}`)
@@ -44,21 +47,96 @@ dgenies.result.export.export_paf = function () {
     document.getElementById('my-download').click();
 };
 
+dgenies.result.export.dl_fasta = function (gzip=false) {
+    let export_div = $("div#export-pict");
+    export_div.html("");
+    export_div.append($("<a>").attr("href", `/fasta-query/${dgenies.result.id_res}`)
+        .attr("download", d3.boxplot.name_y + (gzip ? ".fasta.gz" : ".fasta")).attr("id", "my-download").text("download"));
+    dgenies.hide_loading();
+    document.getElementById('my-download').click();
+};
+
+dgenies.result.export.export_fasta = function(compress=false) {
+    dgenies.show_loading("Building files...", 180);
+    dgenies.post("/get-fasta-query/" + dgenies.result.id_res,
+        {
+            gzip: compress
+        },
+        function(data, success) {
+            console.log(data);
+            if (data["status"] === 0) {
+                window.setTimeout(() => {
+                    dgenies.result.export.export_fasta();
+                }, 10000)
+            }
+            else if (data["status"] === 2) {
+                dgenies.result.export.dl_fasta(data["gzip"]);
+            }
+            else if (data["status"] === 1) {
+                dgenies.hide_loading();
+                dgenies.notify("We are building your Fasta file. You will receive by mail a link to download it soon!",
+                    "info");
+            }
+            else {
+                dgenies.hide_loading();
+                dgenies.notify("An error has occurred. Please contact us to report the bug", "danger");
+            }
+        });
+};
+
+dgenies.result.export.ask_export_fasta = function () {
+    let dialog = $("<div>")
+        .attr("id", "dialog-confirm")
+        .attr("title", "Gzip?");
+    let icon = $("<span>")
+        .attr("class", "ui-icon ui-icon-help")
+        .css("float", "left")
+        .css("margin", "12px 12px 20px 0");
+    let body = $("<p>");
+    body.append(icon);
+    body.append("Compression is recommanded on slow connections. Download Gzip file?");
+    dialog.append(body);
+    dialog.dialog({
+        resizable: false,
+        height: "auto",
+        width: 500,
+        modal: true,
+        buttons: {
+            "Use default": function() {
+                $( this ).dialog( "close" );
+                dgenies.result.export.export_fasta(false);
+            },
+            "Use Gzip": function () {
+                $( this ).dialog( "close" );
+                dgenies.result.export.export_fasta(true);
+            },
+            Cancel: function () {
+                $( this ).dialog( "close" );
+            }
+        }
+    })
+};
+
 dgenies.result.export.export = function () {
     let select = $("form#export select");
     let selection = parseInt(select.val());
-    dgenies.show_loading("Building files...", 180);
     window.setTimeout(() => {
         if (selection > 0) {
+            let async = false;
             if (selection === 1)
                 dgenies.result.export.export_svg();
             else if (selection === 2)
                 dgenies.result.export.export_png();
             else if (selection === 3)
                 dgenies.result.export.export_paf();
+            else if (selection === 4) {
+                dgenies.result.export.ask_export_fasta();
+                async = true;
+            }
             else
                 dgenies.notify("Not supported yet!", "danger", 2000);
-            dgenies.hide_loading();
+            if (!async)
+                dgenies.hide_loading();
             select.val("0");
         }
     }, 0);

lib/functions.py

@@ -3,8 +3,16 @@ import random
 import string
 import gzip
 import io
+import shutil
 import re
+import traceback
 from lib.Fasta import Fasta
+from collections import OrderedDict
+from Bio import SeqIO
+from jinja2 import Template
+from config_reader import AppConfigReader
+from database import Job
+from pony.orm import db_session
 
 ALLOWED_EXTENSIONS = ['fa', 'fasta', 'fna', 'fa.gz', 'fasta.gz', 'fna.gz']
 
@@ -56,4 +64,141 @@ class Functions:
                     elif len(line) > 0:
                         len_c += len(line)
                 if contig is not None and len_c > 0:
-                    out_file.write("%s\t%d\n" % (contig, len_c))
\ No newline at end of file
+                    out_file.write("%s\t%d\n" % (contig, len_c))
+
+    @staticmethod
+    def __get_do_sort(fasta, is_sorted):
+        do_sort = False
+        if is_sorted:
+            do_sort = True
+            if fasta.endswith(".sorted"):
+                do_sort = False
+        return do_sort
+
+    @staticmethod
+    def get_fasta_file(res_dir, type_f, is_sorted):
+        fasta_file = None
+        try:
+            with open(os.path.join(res_dir, "." + type_f), "r") as save_name:
+                fasta_file = save_name.readline()
+        except IOError:
+            print(res_dir + ": Unable to load saved name for " + type_f)
+            pass
+        if fasta_file is not None and os.path.exists(fasta_file):
+            fasta_file_uc = fasta_file
+            if fasta_file.endswith(".gz"):
+                fasta_file_uc = fasta_file[:-3]
+            if is_sorted:
+                sorted_fasta = fasta_file_uc + ".sorted"
+                if os.path.exists(sorted_fasta):
+                    fasta_file = sorted_fasta
+                else:
+                    sorted_fasta = fasta_file_uc + ".gz.sorted"
+                    if os.path.exists(sorted_fasta):
+                        fasta_file = sorted_fasta
+
+        return fasta_file
+
+    @staticmethod
+    def uncompress(filename):
+        try:
+            uncompressed = filename.rsplit('.', 1)[0]
+            parts = uncompressed.rsplit("/", 1)
+            file_path = parts[0]
+            basename = parts[1]
+            n = 2
+            while os.path.exists(uncompressed):
+                uncompressed = "%s/%d_%s" % (file_path, n, basename)
+                n += 1
+            with open(filename, "rb") as infile, open(uncompressed, "wb") as outfile:
+                outfile.write(gzip.decompress(infile.read()))
+            return uncompressed
+        except Exception as e:
+            print(traceback.format_exc())
+            return None
+
+    @staticmethod
+    def compress(filename):
+        try:
+            if not filename.endswith(".gz") and not filename.endswith(".gz.sorted"):
+                compressed = filename + ".gz" if not filename.endswith(".sorted") else filename[:-7] + ".gz.sorted"
+                parts = compressed.rsplit("/", 1)
+                file_path = parts[0]
+                basename = parts[1]
+                n = 2
+                while os.path.exists(compressed):
+                    compressed = "%s/%d_%s" % (file_path, n, basename)
+                    n += 1
+                with open(filename, "rb") as infile, gzip.open(compressed, "wb") as outfile:
+                    shutil.copyfileobj(infile, outfile)
+                os.remove(filename)
+                return compressed
+            return filename
+        except Exception as e:
+            print(traceback.format_exc())
+            return None
+
+    @staticmethod
+    def read_index(index_file):
+        index = OrderedDict()
+        with open(index_file, "r") as index_f:
+            lines = index_f.readlines()
+            for line in lines[1:]:
+                if line != "":
+                    parts = line.strip("\n").split("\t")
+                    name = parts[0]
+                    lenght = int(parts[1])
+                    to_reverse = parts[2] == "1" if len(parts) >= 3 else False
+                    index[name] = {
+                        "length": lenght,
+                        "to_reverse": to_reverse
+                    }
+        return index
+
+    @staticmethod
+    @db_session
+    def get_mail_for_job(id_job):
+        j1 = Job.get(id_job=id_job)
+        return j1.email
+
+    @staticmethod
+    def send_fasta_ready(mailer, job_name):
+        config_reader = AppConfigReader()
+        web_url = config_reader.get_web_url()
+        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "mail_templates", "dl_fasta.html")) \
+                as t_file:
+            template = Template(t_file.read())
+            message_html = template.render(job_name=job_name, status="success", url_base=web_url)
+        message = "D-Genies\n\n" \
+                  "Job %s - Download fasta\n\n" % job_name
+        message += "Query fasta file for job %s is ready to download." % job_name
+        message += "You can click on the link below to download it:\n\n"
+        message += "%s/fasta-query/%s" % (web_url, job_name)
+        mailer.send_mail([Functions.get_mail_for_job(job_name)], "Job %s - Download fasta" % job_name, message,
+                         message_html)
+
+    @staticmethod
+    def sort_fasta(job_name, fasta_file, index_file, lock_file, compress=False, mailer=None):
+        print("Loading index...")
+        index = Functions.read_index(index_file)
+        print("Starting fasta sort...")
+        is_compressed = fasta_file.endswith(".gz")
+        if is_compressed:
+            fasta_file = Functions.uncompress(fasta_file)
+        seq = SeqIO.index(fasta_file, "fasta")
+        fasta_file_o = fasta_file + ".sorted"
+        with open(fasta_file_o, "w") as fasta_out:
+            for name, props in index.items():
+                sequence = seq[name]
+                if props["to_reverse"]:
+                    sequence = sequence[::-1]
+                SeqIO.write(sequence, fasta_out, "fasta")
+        if is_compressed:
+            os.remove(fasta_file)
+        if compress:
+            print("Compress...")
+            Functions.compress(fasta_file_o)
+        os.remove(lock_file)
+        if mailer is not None:
+            Functions.send_fasta_ready(mailer, job_name)
+        print("Fasta sort done!")

lib/job_manager.py

@@ -100,14 +100,22 @@ class JobManager:
         db.commit()
         return status == "success"
 
-    def __getting_local_file(self, fasta: Fasta):
-        finale_path = os.path.join(self.output_dir, os.path.basename(fasta.get_path()))
+    def __getting_local_file(self, fasta: Fasta, type_f):
+        finale_path = os.path.join(self.output_dir, type_f + "_" + os.path.basename(fasta.get_path()))
         shutil.move(fasta.get_path(), finale_path)
+        with open(os.path.join(self.output_dir, "." + type_f), "w") as save_file:
+            save_file.write(finale_path)
         return finale_path
 
-    def __getting_file_from_url(self, fasta: Fasta):
-        finale_path = wget.download(fasta.get_path(), self.output_dir, None)
-        return finale_path
+    def __getting_file_from_url(self, fasta: Fasta, type_f):
+        dl_path = wget.download(fasta.get_path(), self.output_dir, None)
+        filename = os.path.basename(dl_path)
+        name = os.path.splitext(filename.replace(".gz", ""))[0]
+        finale_path = os.path.join(self.output_dir, type_f + "_" + filename)
+        shutil.move(dl_path, finale_path)
+        with open(os.path.join(self.output_dir, "." + type_f), "w") as save_file:
+            save_file.write(finale_path)
+        return finale_path, name
 
     @db_session
     def __check_url(self, fasta: Fasta):
@@ -143,20 +151,18 @@ class JobManager:
         correct = True
         if self.query is not None:
             if self.query.get_type() == "local":
-                self.query.set_path(self.__getting_local_file(self.query))
+                self.query.set_path(self.__getting_local_file(self.query, "query"))
             elif self.__check_url(self.query):
-                finale_path = self.__getting_file_from_url(self.query)
-                filename = os.path.splitext(os.path.basename(finale_path).replace(".gz", ""))[0]
+                finale_path, filename = self.__getting_file_from_url(self.query)
                 self.query.set_path(finale_path)
                 self.query.set_name(filename)
             else:
                 correct = False
         if correct and self.target is not None:
             if self.target.get_type() == "local":
-                self.target.set_path(self.__getting_local_file(self.target))
+                self.target.set_path(self.__getting_local_file(self.target, "target"))
             elif self.__check_url(self.target):
-                finale_path = self.__getting_file_from_url(self.target)
-                filename = os.path.splitext(os.path.basename(finale_path).replace(".gz", ""))[0]
+                finale_path, filename = self.__getting_file_from_url(self.target)
                 self.target.set_path(finale_path)
                 self.target.set_name(filename)
             else:

lib/mail_templates/dl_fasta.html

+<style>
+    .inline {
+        display: inline-block;
+        vertical-align: middle;
+    }
+
+    .header {
+        background: #21264a;
+        padding: 5px;
+    }
+
+    .header h1 {
+        margin: 0;
+        color: white;
+    }
+</style>
+
+<div class="header">
+    <img src="{{ url_base }}/static/images/logo.png" height="45px" alt="" class="inline" style="margin-right: 5px;"/>
+    <h1 class="inline">D-Genies</h1>
+</div>
+<h3>
+    {% if status == "success" %}
+        Job {{ job_name }} - Download fasta
+    {% else %}
+        Job {{ job_name }} - Build of fasta failed
+    {% endif %}
+</h3>
+<p>Hi,</p>
+{% if status == "success" %}
+    <p>Query fasta file for job {{ job_name }} is ready to download.
+        You can <a href="{{ url_base }}/fasta-query/{{ job_name }}">click here</a> to download it.</p>
+{% else %}
+    <p>Build of query fasta file for job {{ job_name }} has failed. You can try again. If the problem persists, please contact the support.</p>
+{% endif %}
+<p>See you soon on D-Genies,</p>
+<p>The D-Genies team</p>
\ No newline at end of file

lib/mailer.py

@@ -11,18 +11,34 @@ class Mailer:
         self.mail_status = config_reader.get_mail_status_sender()
         self.mail_reply = config_reader.get_mail_reply()
         self.mail_org = config_reader.get_mail_org()
+        self.disable = config_reader.get_disable_mail()
 
     def __send_async_email(self, msg):
         with self.app.app_context():
             self.mail.send(msg)
 
     def send_mail(self, recipients: list, subject: str, message: str, message_html: str=None):
-        msg = Message(
-            subject= subject,
-            recipients=recipients,
-            html=message_html,
-            body=message,
-            sender=(self.mail_org, self.mail_status) if self.mail_org is not None else self.mail_status,
-            reply_to=self.mail_reply
-        )
-        self.__send_async_email(msg)
+        sender = (self.mail_org, self.mail_status) if self.mail_org is not None else self.mail_status
+        reply = self.mail_reply
+        if not self.disable:
+            msg = Message(
+                subject= subject,
+                recipients=recipients,
+                html=message_html,
+                body=message,
+                sender=sender,
+                reply_to=reply
+            )
+            self.__send_async_email(msg)
+        else:  # Print debug
+            print("################\n"
+                  "# WARNING !!!! #\n"
+                  "################\n\n"
+                  "!!! SEND MAILS DISABLED BY CONFIGURATION !!!\n\n"
+                  "(This might be disabled in production)\n\n")
+            print("Sender: %s <%s>\n" % sender)
+            print("Reply to: %s\n" % reply)
+            print("Recipients: %s\n" % ", ".join(recipients))
+            print("Subject: %s\n" % subject)
+            print("Message:\n%s\n\n" % message)
+            print("Message HTML:\n%s\n\n" % message_html)

lib/paf.py

@@ -3,6 +3,7 @@
 import os
 from math import sqrt
 from numpy import mean
+from pathlib import Path
 
 
 class Paf:
@@ -14,7 +15,7 @@ class Paf:
         self.idx_q = idx_q
         self.idx_t = idx_t
         self.sorted = False
-        if os.path.exists(paf + ".sorted") and os.path.exists(idx_q + ".sorted"):
+        if os.path.exists(os.path.join(os.path.dirname(paf), ".sorted")):
             self.paf += ".sorted"
             self.idx_q += ".sorted"
             self.sorted = True
@@ -273,6 +274,15 @@ class Paf:
                 idx.write("\t".join([contig, str(self.q_contigs[contig]), "1" if contig in contigs_reoriented else "0"])
                           + "\n")
 
+    def set_sorted(self, is_sorted):
+        self.sorted = is_sorted
+        sorted_touch = os.path.join(os.path.dirname(self.paf), ".sorted")
+        if is_sorted:
+            Path(sorted_touch).touch()
+        else:
+            if os.path.exists(sorted_touch):
+                os.remove(sorted_touch)
+
     def sort(self):
         self.parse_paf(False)
         if not self.sorted:  # Do the sort
@@ -351,7 +361,7 @@ class Paf:
             # Update index:
             self._update_query_index(reorient_contigs)
 
-            self.sorted = True
+            self.set_sorted(True)
 
         else:  # Undo the sort
             if os.path.exists(self.paf):
@@ -360,7 +370,7 @@ class Paf:
             if os.path.exists(self.idx_q):
                 os.remove(self.idx_q)
                 self.idx_q = self.idx_q.replace(".sorted", "")
-            self.sorted = False
+            self.set_sorted(False)
 
         # Re parse PAF file:
         self.parsed = False

requirements.txt

@@ -6,3 +6,4 @@ docopt==0.6.*
 numpy
 wget==3.2
 requests==2.18.*
+biopython==1.70

srv/config_reader.py

@@ -87,6 +87,12 @@ class AppConfigReader(object):
 
     def get_send_mail_status(self):
         try:
-            return self.replace_vars(self.reader.get("mail", "send_mail_status")).lower() == "true"
+            return self.reader.get("mail", "send_mail_status").lower() == "true"
+        except NoOptionError:
+            return True
+
+    def get_disable_mail(self):
+        try:
+            return self.reader.get("mail", "disable").lower() == "true"
         except NoOptionError:
             return False

srv/main.py

@@ -5,8 +5,9 @@ import time
 import datetime
 import shutil
 import re
+import threading
 from flask import Flask, render_template, request, url_for, jsonify, session, Response, abort
-from flask_mail import Mail
+from pathlib import Path
 from lib.paf import Paf
 from config_reader import AppConfigReader
 from lib.job_manager import JobManager
@@ -147,14 +148,14 @@ def result(id_res):
     return response
 
 
-def get_file(file):  # pragma: no cover
+def get_file(file, gzip=False):  # pragma: no cover
     try:
         # Figure out how flask returns static files
         # Tried:
         # - render_template
         # - send_file
         # This should not be so non-obvious
-        return open(file).read()
+        return open(file, "rb" if gzip else "r").read()
     except IOError as exc:
         return str(exc)
 
@@ -201,6 +202,55 @@ def sort_graph(id_res):
     return jsonify({"success": False, "message": paf.error})
 
 
+@app.route('/get-fasta-query/<id_res>', methods=['POST'])
+def build_fasta(id_res):
+    res_dir = os.path.join(app_data, id_res)
+    lock_query = os.path.join(res_dir, ".query-fasta-build")
+    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
+    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
+    if query_fasta is not None:
+        if is_sorted and not query_fasta.endswith(".sorted"):
+            # Do the sort
+            Path(lock_query).touch()
+            thread = threading.Timer(1, Functions.sort_fasta, kwargs={
+                "job_name": id_res,
+                "fasta_file": query_fasta,
+                "index_file": os.path.join(res_dir, "query.idx.sorted"),
+                "lock_file": lock_query,
+                "compress": request.form["gzip"],
+                "mailer": mailer
+            })
+            thread.start()
+            return jsonify({"success": True, "status": 0, "status_message": "Started"})
+        elif is_sorted and os.path.exists(lock_query):
+            # Sort is already in progress
+            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
+        else:
+            # No sort to do or sort done
+            return jsonify({"success": True, "status": 2, "status_message": "Done",
+                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
+    else:
+        return jsonify({"success": False,
+                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})
+
+
+@app.route('/fasta-query/<id_res>', methods=['GET'])
+def dl_fasta(id_res):
+    res_dir = os.path.join(app_data, id_res)
+    lock_query = os.path.join(res_dir, ".query-fasta-build")
+    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
+    if not os.path.exists(lock_query) or not is_sorted:
+        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
+        print(query_fasta)
+        if query_fasta is not None:
+            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
+                content = get_file(query_fasta, True)
+                return Response(content, mimetype="application/gzip")
+            content = get_file(query_fasta)
+            return Response(content, mimetype="text/plain")
+    abort(404)
+
+
 @app.route("/upload", methods=['POST'])
 def upload():
     if "user_tmp_dir" in session and session["user_tmp_dir"] != "":

srv/templates/results.html

@@ -39,7 +39,7 @@
                 <option value="1">Svg</option>
                 <option value="2">Png</option>
                 <option value="3">Paf file</option>
-                <option value="4">Fasta files</option>
+                <option value="4">Query Fasta</option>
             </select>
         </form>
     </div>